2024-11-12 07:47:25,497 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 07:47:25,509 main DEBUG Took 0.009302 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 07:47:25,509 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 07:47:25,510 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 07:47:25,510 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 07:47:25,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,518 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 07:47:25,530 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,531 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,532 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,532 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,533 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,534 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,534 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,535 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,535 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,536 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,536 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,537 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,537 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,538 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,538 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,538 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,539 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,539 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,539 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,540 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,540 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,540 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,541 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 07:47:25,541 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,541 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 07:47:25,543 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 07:47:25,544 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 07:47:25,546 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 07:47:25,546 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 07:47:25,547 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 07:47:25,548 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 07:47:25,556 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 07:47:25,558 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 07:47:25,560 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 07:47:25,560 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 07:47:25,561 main DEBUG createAppenders(={Console}) 2024-11-12 07:47:25,561 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-12 07:47:25,562 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-12 07:47:25,562 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-12 07:47:25,563 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 07:47:25,563 main DEBUG OutputStream closed 2024-11-12 07:47:25,563 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 07:47:25,564 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 07:47:25,564 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-12 07:47:25,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 07:47:25,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 07:47:25,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 07:47:25,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 07:47:25,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 07:47:25,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 07:47:25,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 07:47:25,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 07:47:25,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 07:47:25,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 07:47:25,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 07:47:25,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 07:47:25,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 07:47:25,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 07:47:25,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 07:47:25,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 07:47:25,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 07:47:25,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 07:47:25,646 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 07:47:25,646 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-12 07:47:25,647 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 07:47:25,648 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-12T07:47:25,860 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4 2024-11-12 07:47:25,863 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 07:47:25,864 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T07:47:25,873 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-12T07:47:25,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T07:47:25,919 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/cluster_8b863ce8-e81b-609c-c8dc-bd60e08d8037, deleteOnExit=true 2024-11-12T07:47:25,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T07:47:25,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/test.cache.data in system properties and HBase conf 2024-11-12T07:47:25,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T07:47:25,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/hadoop.log.dir in system properties and HBase conf 2024-11-12T07:47:25,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T07:47:25,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T07:47:25,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T07:47:26,027 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T07:47:26,114 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T07:47:26,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T07:47:26,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T07:47:26,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T07:47:26,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T07:47:26,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T07:47:26,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T07:47:26,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T07:47:26,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T07:47:26,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T07:47:26,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/nfs.dump.dir in system properties and HBase conf 2024-11-12T07:47:26,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/java.io.tmpdir in system properties and HBase conf 2024-11-12T07:47:26,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T07:47:26,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T07:47:26,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T07:47:27,136 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T07:47:27,209 INFO [Time-limited test {}] log.Log(170): Logging initialized @2471ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T07:47:27,277 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T07:47:27,334 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T07:47:27,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T07:47:27,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T07:47:27,357 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T07:47:27,368 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T07:47:27,370 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/hadoop.log.dir/,AVAILABLE} 2024-11-12T07:47:27,372 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T07:47:27,551 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/java.io.tmpdir/jetty-localhost-38197-hadoop-hdfs-3_4_1-tests_jar-_-any-12355979585855730785/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T07:47:27,557 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:38197} 2024-11-12T07:47:27,558 INFO [Time-limited test {}] server.Server(415): Started @2820ms 2024-11-12T07:47:28,056 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T07:47:28,063 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T07:47:28,064 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T07:47:28,065 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T07:47:28,065 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T07:47:28,065 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/hadoop.log.dir/,AVAILABLE} 2024-11-12T07:47:28,066 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T07:47:28,164 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/java.io.tmpdir/jetty-localhost-35775-hadoop-hdfs-3_4_1-tests_jar-_-any-14869517611311302475/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T07:47:28,164 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:35775} 2024-11-12T07:47:28,165 INFO [Time-limited test {}] server.Server(415): Started @3427ms 2024-11-12T07:47:28,211 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T07:47:29,016 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/cluster_8b863ce8-e81b-609c-c8dc-bd60e08d8037/data/data2/current/BP-611431002-172.17.0.2-1731397646649/current, will proceed with Du for space computation calculation, 2024-11-12T07:47:29,016 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/cluster_8b863ce8-e81b-609c-c8dc-bd60e08d8037/data/data1/current/BP-611431002-172.17.0.2-1731397646649/current, will proceed with Du for space computation calculation, 2024-11-12T07:47:29,042 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T07:47:29,090 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5b632b62258579f9 with lease ID 0x80d63005fa9ffb55: Processing first storage report for DS-c7e5db9b-d552-48dd-86ef-bdaba41dcfd2 from datanode DatanodeRegistration(127.0.0.1:40253, datanodeUuid=e2a7e571-c659-4a13-8f26-ad75541ae184, infoPort=46801, infoSecurePort=0, ipcPort=33391, storageInfo=lv=-57;cid=testClusterID;nsid=372269416;c=1731397646649) 2024-11-12T07:47:29,092 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5b632b62258579f9 with lease ID 0x80d63005fa9ffb55: from storage DS-c7e5db9b-d552-48dd-86ef-bdaba41dcfd2 node DatanodeRegistration(127.0.0.1:40253, datanodeUuid=e2a7e571-c659-4a13-8f26-ad75541ae184, infoPort=46801, infoSecurePort=0, ipcPort=33391, storageInfo=lv=-57;cid=testClusterID;nsid=372269416;c=1731397646649), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T07:47:29,093 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5b632b62258579f9 with lease ID 0x80d63005fa9ffb55: Processing first storage report for DS-a1773d73-e72d-402e-804a-8b8d64c0e516 from datanode DatanodeRegistration(127.0.0.1:40253, datanodeUuid=e2a7e571-c659-4a13-8f26-ad75541ae184, infoPort=46801, infoSecurePort=0, ipcPort=33391, storageInfo=lv=-57;cid=testClusterID;nsid=372269416;c=1731397646649) 2024-11-12T07:47:29,093 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5b632b62258579f9 with lease ID 0x80d63005fa9ffb55: from storage DS-a1773d73-e72d-402e-804a-8b8d64c0e516 node DatanodeRegistration(127.0.0.1:40253, datanodeUuid=e2a7e571-c659-4a13-8f26-ad75541ae184, infoPort=46801, infoSecurePort=0, ipcPort=33391, storageInfo=lv=-57;cid=testClusterID;nsid=372269416;c=1731397646649), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T07:47:29,132 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4 2024-11-12T07:47:29,196 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/cluster_8b863ce8-e81b-609c-c8dc-bd60e08d8037/zookeeper_0, clientPort=63742, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/cluster_8b863ce8-e81b-609c-c8dc-bd60e08d8037/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/cluster_8b863ce8-e81b-609c-c8dc-bd60e08d8037/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T07:47:29,205 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63742 2024-11-12T07:47:29,219 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:47:29,223 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:47:29,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741825_1001 (size=7) 2024-11-12T07:47:29,859 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f with version=8 2024-11-12T07:47:29,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/hbase-staging 2024-11-12T07:47:29,940 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T07:47:30,176 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:47:30,185 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:47:30,186 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:47:30,190 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:47:30,190 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:47:30,190 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:47:30,323 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T07:47:30,379 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T07:47:30,388 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T07:47:30,392 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:47:30,415 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30674 (auto-detected) 2024-11-12T07:47:30,416 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T07:47:30,432 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43319 2024-11-12T07:47:30,452 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43319 connecting to ZooKeeper ensemble=127.0.0.1:63742 2024-11-12T07:47:30,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433190x0, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:47:30,592 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43319-0x1012f4d15df0000 connected 2024-11-12T07:47:30,691 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:47:30,694 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:47:30,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:47:30,707 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f, hbase.cluster.distributed=false 2024-11-12T07:47:30,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:47:30,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43319 2024-11-12T07:47:30,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43319 2024-11-12T07:47:30,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43319 2024-11-12T07:47:30,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43319 2024-11-12T07:47:30,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43319 2024-11-12T07:47:30,827 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f9fc96254ef8:0 server-side Connection retries=45 2024-11-12T07:47:30,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:47:30,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T07:47:30,829 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T07:47:30,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T07:47:30,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T07:47:30,832 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T07:47:30,834 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T07:47:30,835 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44299 2024-11-12T07:47:30,837 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44299 connecting to ZooKeeper ensemble=127.0.0.1:63742 2024-11-12T07:47:30,838 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:47:30,841 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:47:30,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442990x0, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T07:47:30,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442990x0, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:47:30,867 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44299-0x1012f4d15df0001 connected 2024-11-12T07:47:30,872 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T07:47:30,880 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T07:47:30,882 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T07:47:30,887 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T07:47:30,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44299 2024-11-12T07:47:30,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44299 2024-11-12T07:47:30,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44299 2024-11-12T07:47:30,892 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44299 2024-11-12T07:47:30,893 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44299 2024-11-12T07:47:30,908 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f9fc96254ef8:43319 2024-11-12T07:47:30,909 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:30,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:47:30,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:47:30,950 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:30,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:30,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T07:47:30,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:30,992 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T07:47:30,994 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f9fc96254ef8,43319,1731397650023 from backup master directory 2024-11-12T07:47:31,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:47:31,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:31,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T07:47:31,005 WARN [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:47:31,005 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:31,007 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T07:47:31,008 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T07:47:31,057 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/hbase.id] with ID: 402938bb-bb5a-4f15-b60d-61d4ad603f05 2024-11-12T07:47:31,057 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/.tmp/hbase.id 2024-11-12T07:47:31,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741826_1002 (size=42) 2024-11-12T07:47:31,469 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/.tmp/hbase.id]:[hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/hbase.id] 2024-11-12T07:47:31,521 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T07:47:31,526 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T07:47:31,543 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-12T07:47:31,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:31,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:31,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741827_1003 (size=196) 2024-11-12T07:47:32,041 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:47:32,044 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T07:47:32,057 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:32,060 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:47:32,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741828_1004 (size=1189) 2024-11-12T07:47:32,509 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store 2024-11-12T07:47:32,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741829_1005 (size=34) 2024-11-12T07:47:32,958 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T07:47:32,961 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:47:32,962 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T07:47:32,963 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:47:32,963 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:47:32,964 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T07:47:32,965 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:47:32,965 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:47:32,966 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731397652962Disabling compacts and flushes for region at 1731397652962Disabling writes for close at 1731397652964 (+2 ms)Writing region close event to WAL at 1731397652965 (+1 ms)Closed at 1731397652965 2024-11-12T07:47:32,968 WARN [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/.initializing 2024-11-12T07:47:32,968 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/WALs/f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:32,976 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:47:32,990 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f9fc96254ef8%2C43319%2C1731397650023, suffix=, logDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/WALs/f9fc96254ef8,43319,1731397650023, archiveDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/oldWALs, maxLogs=10 2024-11-12T07:47:33,012 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/WALs/f9fc96254ef8,43319,1731397650023/f9fc96254ef8%2C43319%2C1731397650023.1731397652994, exclude list is [], retry=0 2024-11-12T07:47:33,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40253,DS-c7e5db9b-d552-48dd-86ef-bdaba41dcfd2,DISK] 2024-11-12T07:47:33,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T07:47:33,059 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/WALs/f9fc96254ef8,43319,1731397650023/f9fc96254ef8%2C43319%2C1731397650023.1731397652994 2024-11-12T07:47:33,060 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46801:46801)] 2024-11-12T07:47:33,060 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:47:33,061 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:47:33,064 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,065 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,099 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T07:47:33,125 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:33,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:33,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T07:47:33,131 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:33,132 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:47:33,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T07:47:33,135 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:33,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:47:33,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T07:47:33,139 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:33,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:47:33,140 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,144 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,145 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,150 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,151 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,154 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T07:47:33,157 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T07:47:33,161 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:47:33,163 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66523661, jitterRate=-0.00872020423412323}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T07:47:33,168 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731397653076Initializing all the Stores at 1731397653078 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397653079 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397653079Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397653080 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397653080Cleaning up temporary data from old regions at 1731397653151 (+71 ms)Region opened successfully at 1731397653168 (+17 ms) 2024-11-12T07:47:33,169 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T07:47:33,198 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74e0cf90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:47:33,225 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T07:47:33,234 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T07:47:33,234 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T07:47:33,236 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T07:47:33,237 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T07:47:33,242 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T07:47:33,242 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T07:47:33,265 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T07:47:33,273 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T07:47:33,324 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T07:47:33,328 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T07:47:33,330 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T07:47:33,340 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T07:47:33,343 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T07:47:33,347 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T07:47:33,357 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T07:47:33,359 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T07:47:33,371 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T07:47:33,393 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T07:47:33,404 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T07:47:33,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:47:33,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T07:47:33,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:33,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:33,420 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f9fc96254ef8,43319,1731397650023, sessionid=0x1012f4d15df0000, setting cluster-up flag (Was=false) 2024-11-12T07:47:33,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:33,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:33,474 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T07:47:33,477 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:33,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:33,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:33,533 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T07:47:33,537 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:33,549 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T07:47:33,600 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(746): ClusterId : 402938bb-bb5a-4f15-b60d-61d4ad603f05 2024-11-12T07:47:33,602 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T07:47:33,621 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T07:47:33,626 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T07:47:33,627 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T07:47:33,631 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T07:47:33,638 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T07:47:33,639 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T07:47:33,640 DEBUG [RS:0;f9fc96254ef8:44299 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b2c56e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f9fc96254ef8/172.17.0.2:0 2024-11-12T07:47:33,643 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f9fc96254ef8,43319,1731397650023 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T07:47:33,650 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:47:33,651 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:47:33,651 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:47:33,651 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=5, maxPoolSize=5 2024-11-12T07:47:33,652 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f9fc96254ef8:0, corePoolSize=10, maxPoolSize=10 2024-11-12T07:47:33,652 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,652 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:47:33,652 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,654 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731397683653 2024-11-12T07:47:33,655 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T07:47:33,657 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f9fc96254ef8:44299 2024-11-12T07:47:33,657 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T07:47:33,657 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:47:33,658 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T07:47:33,661 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T07:47:33,661 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T07:47:33,661 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T07:47:33,661 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T07:47:33,661 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T07:47:33,662 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T07:47:33,662 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T07:47:33,664 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:33,664 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(2659): reportForDuty to master=f9fc96254ef8,43319,1731397650023 with port=44299, startcode=1731397650797 2024-11-12T07:47:33,663 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,665 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T07:47:33,667 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T07:47:33,668 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T07:47:33,669 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T07:47:33,671 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T07:47:33,671 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T07:47:33,675 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397653673,5,FailOnTimeoutGroup] 2024-11-12T07:47:33,676 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397653676,5,FailOnTimeoutGroup] 2024-11-12T07:47:33,677 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,677 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T07:47:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741831_1007 (size=1321) 2024-11-12T07:47:33,678 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,679 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,679 DEBUG [RS:0;f9fc96254ef8:44299 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T07:47:33,706 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32983, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T07:47:33,712 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43319 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:33,714 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43319 {}] master.ServerManager(517): Registering regionserver=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:33,727 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f 2024-11-12T07:47:33,727 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34321 2024-11-12T07:47:33,727 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T07:47:33,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:47:33,750 DEBUG [RS:0;f9fc96254ef8:44299 {}] zookeeper.ZKUtil(111): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:33,750 WARN [RS:0;f9fc96254ef8:44299 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T07:47:33,750 INFO [RS:0;f9fc96254ef8:44299 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:47:33,751 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/WALs/f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:33,753 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f9fc96254ef8,44299,1731397650797] 2024-11-12T07:47:33,777 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T07:47:33,793 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T07:47:33,797 INFO [RS:0;f9fc96254ef8:44299 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T07:47:33,798 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,799 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T07:47:33,803 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T07:47:33,805 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,805 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,805 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,805 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,805 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,805 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,806 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f9fc96254ef8:0, corePoolSize=2, maxPoolSize=2 2024-11-12T07:47:33,806 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,806 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,806 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,806 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,806 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,807 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f9fc96254ef8:0, corePoolSize=1, maxPoolSize=1 2024-11-12T07:47:33,807 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:47:33,807 DEBUG [RS:0;f9fc96254ef8:44299 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0, corePoolSize=3, maxPoolSize=3 2024-11-12T07:47:33,808 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,808 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,808 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,808 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,808 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,808 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,44299,1731397650797-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:47:33,826 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T07:47:33,828 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,44299,1731397650797-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,828 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,828 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.Replication(171): f9fc96254ef8,44299,1731397650797 started 2024-11-12T07:47:33,847 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:33,848 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1482): Serving as f9fc96254ef8,44299,1731397650797, RpcServer on f9fc96254ef8/172.17.0.2:44299, sessionid=0x1012f4d15df0001 2024-11-12T07:47:33,848 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T07:47:33,849 DEBUG [RS:0;f9fc96254ef8:44299 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:33,849 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,44299,1731397650797' 2024-11-12T07:47:33,849 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T07:47:33,850 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T07:47:33,851 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T07:47:33,851 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T07:47:33,851 DEBUG [RS:0;f9fc96254ef8:44299 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:33,851 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f9fc96254ef8,44299,1731397650797' 2024-11-12T07:47:33,851 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T07:47:33,852 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T07:47:33,853 DEBUG [RS:0;f9fc96254ef8:44299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T07:47:33,853 INFO [RS:0;f9fc96254ef8:44299 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T07:47:33,853 INFO [RS:0;f9fc96254ef8:44299 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T07:47:33,965 INFO [RS:0;f9fc96254ef8:44299 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T07:47:33,969 INFO [RS:0;f9fc96254ef8:44299 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f9fc96254ef8%2C44299%2C1731397650797, suffix=, logDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/WALs/f9fc96254ef8,44299,1731397650797, archiveDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/oldWALs, maxLogs=32 2024-11-12T07:47:33,983 DEBUG [RS:0;f9fc96254ef8:44299 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/WALs/f9fc96254ef8,44299,1731397650797/f9fc96254ef8%2C44299%2C1731397650797.1731397653971, exclude list is [], retry=0 2024-11-12T07:47:33,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40253,DS-c7e5db9b-d552-48dd-86ef-bdaba41dcfd2,DISK] 2024-11-12T07:47:33,991 INFO [RS:0;f9fc96254ef8:44299 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/WALs/f9fc96254ef8,44299,1731397650797/f9fc96254ef8%2C44299%2C1731397650797.1731397653971 2024-11-12T07:47:33,991 DEBUG [RS:0;f9fc96254ef8:44299 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46801:46801)] 2024-11-12T07:47:34,083 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T07:47:34,085 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f 2024-11-12T07:47:34,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741833_1009 (size=32) 2024-11-12T07:47:34,504 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:47:34,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T07:47:34,512 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:47:34,512 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:34,513 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:34,513 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T07:47:34,517 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:47:34,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:34,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:34,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T07:47:34,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:47:34,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:34,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:34,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T07:47:34,528 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:47:34,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:34,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:34,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:47:34,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740 2024-11-12T07:47:34,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740 2024-11-12T07:47:34,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:47:34,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:47:34,535 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:47:34,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:47:34,542 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:47:34,543 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66296998, jitterRate=-0.012097746133804321}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:47:34,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731397654504Initializing all the Stores at 1731397654508 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397654508Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397654509 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397654509Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397654509Cleaning up temporary data from old regions at 1731397654534 (+25 ms)Region opened successfully at 1731397654545 (+11 ms) 2024-11-12T07:47:34,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:47:34,545 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:47:34,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:47:34,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:47:34,546 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:47:34,547 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:47:34,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397654545Disabling compacts and flushes for region at 1731397654545Disabling writes for close at 1731397654546 (+1 ms)Writing region close event to WAL at 1731397654546Closed at 1731397654547 (+1 ms) 2024-11-12T07:47:34,550 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:47:34,550 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T07:47:34,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T07:47:34,564 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:47:34,567 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T07:47:34,723 DEBUG [f9fc96254ef8:43319 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T07:47:34,740 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:34,746 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,44299,1731397650797, state=OPENING 2024-11-12T07:47:34,807 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T07:47:34,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:34,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:47:34,817 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:47:34,817 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:47:34,818 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T07:47:34,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:47:34,995 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T07:47:34,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60025, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T07:47:35,009 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T07:47:35,009 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T07:47:35,010 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T07:47:35,013 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f9fc96254ef8%2C44299%2C1731397650797.meta, suffix=.meta, logDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/WALs/f9fc96254ef8,44299,1731397650797, archiveDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/oldWALs, maxLogs=32 2024-11-12T07:47:35,027 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/WALs/f9fc96254ef8,44299,1731397650797/f9fc96254ef8%2C44299%2C1731397650797.meta.1731397655015.meta, exclude list is [], retry=0 2024-11-12T07:47:35,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40253,DS-c7e5db9b-d552-48dd-86ef-bdaba41dcfd2,DISK] 2024-11-12T07:47:35,033 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/WALs/f9fc96254ef8,44299,1731397650797/f9fc96254ef8%2C44299%2C1731397650797.meta.1731397655015.meta 2024-11-12T07:47:35,034 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46801:46801)] 2024-11-12T07:47:35,034 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:47:35,036 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T07:47:35,038 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T07:47:35,042 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T07:47:35,046 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T07:47:35,047 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:47:35,047 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T07:47:35,047 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T07:47:35,050 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T07:47:35,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T07:47:35,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:35,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:35,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T07:47:35,054 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T07:47:35,054 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:35,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:35,055 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T07:47:35,056 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T07:47:35,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:35,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:35,057 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T07:47:35,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T07:47:35,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:35,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T07:47:35,060 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T07:47:35,062 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740 2024-11-12T07:47:35,065 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740 2024-11-12T07:47:35,067 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T07:47:35,067 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T07:47:35,068 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:47:35,071 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T07:47:35,073 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69585264, jitterRate=0.036901235580444336}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:47:35,073 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T07:47:35,074 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731397655047Writing region info on filesystem at 1731397655048 (+1 ms)Initializing all the Stores at 1731397655049 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397655050 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397655050Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397655050Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731397655050Cleaning up temporary data from old regions at 1731397655067 (+17 ms)Running coprocessor post-open hooks at 1731397655073 (+6 ms)Region opened successfully at 1731397655074 (+1 ms) 2024-11-12T07:47:35,081 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731397654987 2024-11-12T07:47:35,091 DEBUG [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T07:47:35,091 INFO [RS_OPEN_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T07:47:35,093 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:35,095 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f9fc96254ef8,44299,1731397650797, state=OPEN 2024-11-12T07:47:35,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:47:35,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T07:47:35,152 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:47:35,152 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T07:47:35,153 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:35,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T07:47:35,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f9fc96254ef8,44299,1731397650797 in 333 msec 2024-11-12T07:47:35,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T07:47:35,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 610 msec 2024-11-12T07:47:35,174 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T07:47:35,175 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T07:47:35,191 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:35,192 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:35,209 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:35,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45735, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:35,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6550 sec 2024-11-12T07:47:35,234 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731397655234, completionTime=-1 2024-11-12T07:47:35,237 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T07:47:35,237 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T07:47:35,262 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-12T07:47:35,262 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731397715262 2024-11-12T07:47:35,262 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731397775262 2024-11-12T07:47:35,262 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-12T07:47:35,264 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43319,1731397650023-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:35,265 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43319,1731397650023-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:35,265 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43319,1731397650023-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:35,266 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f9fc96254ef8:43319, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:35,266 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:35,267 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:35,273 DEBUG [master/f9fc96254ef8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T07:47:35,295 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.289sec 2024-11-12T07:47:35,296 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T07:47:35,298 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T07:47:35,298 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T07:47:35,299 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T07:47:35,299 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T07:47:35,300 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43319,1731397650023-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T07:47:35,300 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43319,1731397650023-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T07:47:35,307 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T07:47:35,308 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T07:47:35,309 INFO [master/f9fc96254ef8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f9fc96254ef8,43319,1731397650023-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T07:47:35,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:35,311 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T07:47:35,311 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T07:47:35,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:35,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:35,323 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:35,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5670729d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:35,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:35,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:35,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:35,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:35,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:35,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:35,376 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=11, AvailableMemoryMB=6622 2024-11-12T07:47:35,385 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:47:35,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79901794 2024-11-12T07:47:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:47:35,409 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:47:35,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:47:35,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:47:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T07:47:35,430 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:47:35,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-12T07:47:35,434 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:35,437 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:47:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:47:35,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741835_1011 (size=1061) 2024-11-12T07:47:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:47:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:47:35,862 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f 2024-11-12T07:47:35,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741836_1012 (size=53) 2024-11-12T07:47:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:47:36,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:47:36,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 609f0702d0c4b8c7ee5a4c9652b6127b, disabling compactions & flushes 2024-11-12T07:47:36,275 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:36,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:36,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. after waiting 0 ms 2024-11-12T07:47:36,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:36,275 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:36,275 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 609f0702d0c4b8c7ee5a4c9652b6127b: Waiting for close lock at 1731397656275Disabling compacts and flushes for region at 1731397656275Disabling writes for close at 1731397656275Writing region close event to WAL at 1731397656275Closed at 1731397656275 2024-11-12T07:47:36,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:47:36,286 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731397656280"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397656280"}]},"ts":"1731397656280"} 2024-11-12T07:47:36,291 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:47:36,293 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:47:36,295 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397656293"}]},"ts":"1731397656293"} 2024-11-12T07:47:36,299 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T07:47:36,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, ASSIGN}] 2024-11-12T07:47:36,303 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, ASSIGN 2024-11-12T07:47:36,305 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, ASSIGN; state=OFFLINE, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=false 2024-11-12T07:47:36,457 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=609f0702d0c4b8c7ee5a4c9652b6127b, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:36,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, ASSIGN because future has completed 2024-11-12T07:47:36,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:47:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:47:36,629 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:36,629 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:47:36,630 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,630 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:47:36,631 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,631 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,634 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,637 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:47:36,638 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 609f0702d0c4b8c7ee5a4c9652b6127b columnFamilyName A 2024-11-12T07:47:36,638 DEBUG [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:36,639 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.HStore(327): Store=609f0702d0c4b8c7ee5a4c9652b6127b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:47:36,639 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,641 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:47:36,642 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 609f0702d0c4b8c7ee5a4c9652b6127b columnFamilyName B 2024-11-12T07:47:36,642 DEBUG [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:36,642 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.HStore(327): Store=609f0702d0c4b8c7ee5a4c9652b6127b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:47:36,643 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,645 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:47:36,645 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 609f0702d0c4b8c7ee5a4c9652b6127b columnFamilyName C 2024-11-12T07:47:36,645 DEBUG [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:47:36,646 INFO [StoreOpener-609f0702d0c4b8c7ee5a4c9652b6127b-1 {}] regionserver.HStore(327): Store=609f0702d0c4b8c7ee5a4c9652b6127b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:47:36,647 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:36,647 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,649 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,650 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,653 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,653 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,654 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:47:36,657 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,660 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:47:36,661 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 609f0702d0c4b8c7ee5a4c9652b6127b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60065914, jitterRate=-0.10494813323020935}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:47:36,661 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:36,662 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 609f0702d0c4b8c7ee5a4c9652b6127b: Running coprocessor pre-open hook at 1731397656631Writing region info on filesystem at 1731397656631Initializing all the Stores at 1731397656633 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397656633Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397656634 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397656634Cleaning up temporary data from old regions at 1731397656653 (+19 ms)Running coprocessor post-open hooks at 1731397656661 (+8 ms)Region opened successfully at 1731397656662 (+1 ms) 2024-11-12T07:47:36,664 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., pid=6, masterSystemTime=1731397656619 2024-11-12T07:47:36,668 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:36,668 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:36,669 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=609f0702d0c4b8c7ee5a4c9652b6127b, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:36,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:47:36,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T07:47:36,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 in 213 msec 2024-11-12T07:47:36,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T07:47:36,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, ASSIGN in 380 msec 2024-11-12T07:47:36,688 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:47:36,688 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397656688"}]},"ts":"1731397656688"} 2024-11-12T07:47:36,692 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T07:47:36,694 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:47:36,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2730 sec 2024-11-12T07:47:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T07:47:37,590 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:37,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,599 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f6a966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,603 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc7db18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,605 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,615 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:47:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,43319,1731397650023 2024-11-12T07:47:37,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f70fa58 2024-11-12T07:47:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:47:37,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:47:37,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,627 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,631 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39deefb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,637 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ccbe83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,640 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,642 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,644 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,647 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b0e30d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,651 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4545009e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585eb20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,659 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644e9151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ccd238a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,665 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1b5c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb446c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,671 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89bfc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d3547a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,677 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb5cb2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,679 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:47:37,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:47:37,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:47:37,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:47:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:47:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b2908d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:47:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:47:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:37,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:47:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cfd6776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:47:37,690 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,690 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,691 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,690 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,692 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,694 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,694 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,694 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,695 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,695 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,696 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,696 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,697 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,697 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,698 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,706 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,711 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:47:37,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:47:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:47:37,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:47:37,723 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:37,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:47:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-12T07:47:37,736 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:37,739 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:47:37,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:37,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T07:47:37,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:37,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:37,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:37,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:37,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:37,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:47:37,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T07:47:37,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:37,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:37,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:37,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/0eb196bd5eb74510a21bfe6866486c22 is 50, key is test_row_0/A:col10/1731397657792/Put/seqid=0 2024-11-12T07:47:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741837_1013 (size=16681) 2024-11-12T07:47:37,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397717939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397717954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397717960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397717985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397717995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:47:38,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T07:47:38,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397718140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397718141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397718141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397718141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397718142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T07:47:38,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397718358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397718360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397718359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397718360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397718361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:47:38,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/0eb196bd5eb74510a21bfe6866486c22 2024-11-12T07:47:38,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T07:47:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/5ead7a74461b4f6a8c59b119c3b3bb4a is 50, key is test_row_0/B:col10/1731397657792/Put/seqid=0 2024-11-12T07:47:38,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741838_1014 (size=12001) 2024-11-12T07:47:38,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/5ead7a74461b4f6a8c59b119c3b3bb4a 2024-11-12T07:47:38,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/23f9f1359e3b460bb8cd5bd071d36e3e is 50, key is test_row_0/C:col10/1731397657792/Put/seqid=0 2024-11-12T07:47:38,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741839_1015 (size=12001) 2024-11-12T07:47:38,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/23f9f1359e3b460bb8cd5bd071d36e3e 2024-11-12T07:47:38,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T07:47:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/0eb196bd5eb74510a21bfe6866486c22 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/0eb196bd5eb74510a21bfe6866486c22 2024-11-12T07:47:38,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/0eb196bd5eb74510a21bfe6866486c22, entries=250, sequenceid=14, filesize=16.3 K 2024-11-12T07:47:38,620 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-12T07:47:38,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/5ead7a74461b4f6a8c59b119c3b3bb4a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ead7a74461b4f6a8c59b119c3b3bb4a 2024-11-12T07:47:38,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ead7a74461b4f6a8c59b119c3b3bb4a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T07:47:38,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/23f9f1359e3b460bb8cd5bd071d36e3e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/23f9f1359e3b460bb8cd5bd071d36e3e 2024-11-12T07:47:38,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/23f9f1359e3b460bb8cd5bd071d36e3e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T07:47:38,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 609f0702d0c4b8c7ee5a4c9652b6127b in 855ms, sequenceid=14, compaction requested=false 2024-11-12T07:47:38,671 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T07:47:38,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:38,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:38,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T07:47:38,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:38,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:38,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:38,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:38,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:38,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:38,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/f12b0ac12e76435c83af4f09979727fb is 50, key is test_row_0/A:col10/1731397658682/Put/seqid=0 2024-11-12T07:47:38,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T07:47:38,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:38,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741840_1016 (size=14341) 2024-11-12T07:47:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/f12b0ac12e76435c83af4f09979727fb 2024-11-12T07:47:38,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397718732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397718735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397718736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397718744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397718745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/9268c70da2f94ad5a917547b5f466c04 is 50, key is test_row_0/B:col10/1731397658682/Put/seqid=0 2024-11-12T07:47:38,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741841_1017 (size=12001) 2024-11-12T07:47:38,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/9268c70da2f94ad5a917547b5f466c04 2024-11-12T07:47:38,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/983cfdadc7f94ce399979f47167e86d0 is 50, key is test_row_0/C:col10/1731397658682/Put/seqid=0 2024-11-12T07:47:38,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741842_1018 (size=12001) 2024-11-12T07:47:38,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/983cfdadc7f94ce399979f47167e86d0 2024-11-12T07:47:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:47:38,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397718879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397718880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397718880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397718881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397718882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:38,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T07:47:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:38,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:38,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:38,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:38,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/f12b0ac12e76435c83af4f09979727fb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f12b0ac12e76435c83af4f09979727fb 2024-11-12T07:47:38,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f12b0ac12e76435c83af4f09979727fb, entries=200, sequenceid=34, filesize=14.0 K 2024-11-12T07:47:38,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/9268c70da2f94ad5a917547b5f466c04 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/9268c70da2f94ad5a917547b5f466c04 2024-11-12T07:47:38,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/9268c70da2f94ad5a917547b5f466c04, entries=150, sequenceid=34, filesize=11.7 K 2024-11-12T07:47:38,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/983cfdadc7f94ce399979f47167e86d0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/983cfdadc7f94ce399979f47167e86d0 2024-11-12T07:47:38,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/983cfdadc7f94ce399979f47167e86d0, entries=150, sequenceid=34, filesize=11.7 K 2024-11-12T07:47:38,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 609f0702d0c4b8c7ee5a4c9652b6127b in 294ms, sequenceid=34, compaction requested=false 2024-11-12T07:47:38,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:39,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T07:47:39,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:39,057 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:47:39,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:39,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:39,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:39,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:39,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:39,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/4adeb499a63442e88dc9916f219dfd93 is 50, key is test_row_0/A:col10/1731397658734/Put/seqid=0 2024-11-12T07:47:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:39,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:39,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741843_1019 (size=12001) 2024-11-12T07:47:39,109 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/4adeb499a63442e88dc9916f219dfd93 2024-11-12T07:47:39,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/57579a172b1347d8be381413928eafde is 50, key is test_row_0/B:col10/1731397658734/Put/seqid=0 2024-11-12T07:47:39,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741844_1020 (size=12001) 2024-11-12T07:47:39,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397719196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397719196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397719196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397719201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397719204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397719319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397719321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397719321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397719328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397719329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397719529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397719529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397719529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,535 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397719538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397719539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:39,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:39,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/57579a172b1347d8be381413928eafde 2024-11-12T07:47:39,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/a00bbb899fa04979a977965aa976a453 is 50, key is test_row_0/C:col10/1731397658734/Put/seqid=0 2024-11-12T07:47:39,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741845_1021 (size=12001) 2024-11-12T07:47:39,591 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/a00bbb899fa04979a977965aa976a453 2024-11-12T07:47:39,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/4adeb499a63442e88dc9916f219dfd93 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/4adeb499a63442e88dc9916f219dfd93 2024-11-12T07:47:39,633 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/4adeb499a63442e88dc9916f219dfd93, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T07:47:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/57579a172b1347d8be381413928eafde as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/57579a172b1347d8be381413928eafde 2024-11-12T07:47:39,648 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/57579a172b1347d8be381413928eafde, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T07:47:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/a00bbb899fa04979a977965aa976a453 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a00bbb899fa04979a977965aa976a453 2024-11-12T07:47:39,665 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a00bbb899fa04979a977965aa976a453, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T07:47:39,667 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 609f0702d0c4b8c7ee5a4c9652b6127b in 609ms, sequenceid=46, compaction requested=true 2024-11-12T07:47:39,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:39,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:39,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-12T07:47:39,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-12T07:47:39,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T07:47:39,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9310 sec 2024-11-12T07:47:39,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9550 sec 2024-11-12T07:47:39,770 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T07:47:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:39,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-12T07:47:39,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731397659190, max timestamp=1731397659840Num uniques 859; 2024-11-12T07:47:39,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T07:47:39,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:39,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731397659190, max timestamp=1731397659840Num uniques 859; 2024-11-12T07:47:39,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T07:47:39,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397659190, max timestamp=1731397659842Num uniques 859; 2024-11-12T07:47:39,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:47:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:39,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/b90d6d80c5c749f98f6ea21cbe5129fa is 50, key is test_row_0/A:col10/1731397659842/Put/seqid=0 2024-11-12T07:47:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T07:47:39,889 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:39,890 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-12T07:47:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T07:47:39,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:39,896 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:39,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:39,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741846_1022 (size=14341) 2024-11-12T07:47:39,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/b90d6d80c5c749f98f6ea21cbe5129fa 2024-11-12T07:47:39,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/620f91b9c01a4c33b4007efa3dfee026 is 50, key is test_row_0/B:col10/1731397659842/Put/seqid=0 2024-11-12T07:47:39,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397659907, max timestamp=1731397659937Num uniques 859; 2024-11-12T07:47:39,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:47:39,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397659907, max timestamp=1731397659937Num uniques 859; 2024-11-12T07:47:39,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:47:39,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397659907, max timestamp=1731397659936Num uniques 859; 2024-11-12T07:47:39,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:47:39,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741847_1023 (size=12001) 2024-11-12T07:47:39,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/620f91b9c01a4c33b4007efa3dfee026 2024-11-12T07:47:39,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/5f15689b442b47c78ba9e2e1e95cd974 is 50, key is test_row_0/C:col10/1731397659842/Put/seqid=0 2024-11-12T07:47:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T07:47:40,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397720002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397659990, max timestamp=1731397660011Num uniques 1018; 2024-11-12T07:47:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731397659990, max timestamp=1731397660014Num uniques 1018; 2024-11-12T07:47:40,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T07:47:40,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=8516, suffix heap size=133840,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=186 2024-11-12T07:47:40,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731397659990, max timestamp=1731397660014Num uniques 1018; 2024-11-12T07:47:40,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T07:47:40,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741848_1024 (size=12001) 2024-11-12T07:47:40,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731397660066, max timestamp=1731397660100Num uniques 1027; 2024-11-12T07:47:40,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731397660066, max timestamp=1731397660099Num uniques 1045; 2024-11-12T07:47:40,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8102, suffix heap size=136648,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=177 2024-11-12T07:47:40,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T07:47:40,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731397660066, max timestamp=1731397660100Num uniques 1027; 2024-11-12T07:47:40,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-12T07:47:40,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397660117, max timestamp=1731397660175Num uniques 1036; 2024-11-12T07:47:40,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T07:47:40,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397660117, max timestamp=1731397660175Num uniques 1036; 2024-11-12T07:47:40,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T07:47:40,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731397660117, max timestamp=1731397660175Num uniques 1036; 2024-11-12T07:47:40,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-12T07:47:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T07:47:40,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:40,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731397660205, max timestamp=1731397660231Num uniques 1045; 2024-11-12T07:47:40,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-12T07:47:40,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731397660205, max timestamp=1731397660231Num uniques 1045; 2024-11-12T07:47:40,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=15800, suffix heap size=136648,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=345 2024-11-12T07:47:40,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731397660205, max timestamp=1731397660231Num uniques 1009; 2024-11-12T07:47:40,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=15800, suffix heap size=132904,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=345 2024-11-12T07:47:40,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397720278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397720278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397720285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397720297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397720298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397720408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397720409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397720409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397720418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397720418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/5f15689b442b47c78ba9e2e1e95cd974 2024-11-12T07:47:40,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/b90d6d80c5c749f98f6ea21cbe5129fa as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b90d6d80c5c749f98f6ea21cbe5129fa 2024-11-12T07:47:40,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b90d6d80c5c749f98f6ea21cbe5129fa, entries=200, sequenceid=69, filesize=14.0 K 2024-11-12T07:47:40,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/620f91b9c01a4c33b4007efa3dfee026 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/620f91b9c01a4c33b4007efa3dfee026 2024-11-12T07:47:40,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/620f91b9c01a4c33b4007efa3dfee026, entries=150, sequenceid=69, filesize=11.7 K 2024-11-12T07:47:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T07:47:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/5f15689b442b47c78ba9e2e1e95cd974 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5f15689b442b47c78ba9e2e1e95cd974 2024-11-12T07:47:40,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5f15689b442b47c78ba9e2e1e95cd974, entries=150, sequenceid=69, filesize=11.7 K 2024-11-12T07:47:40,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.46 KB/40410, heapSize ~104.20 KB/106704, currentSize=161.02 KB/164880 for 609f0702d0c4b8c7ee5a4c9652b6127b in 699ms, sequenceid=69, compaction requested=true 2024-11-12T07:47:40,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:40,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:40,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:40,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:40,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:40,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:40,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T07:47:40,553 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:40,558 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:40,558 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:40,561 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:47:40,561 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:40,561 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,562 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ead7a74461b4f6a8c59b119c3b3bb4a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/9268c70da2f94ad5a917547b5f466c04, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/57579a172b1347d8be381413928eafde, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/620f91b9c01a4c33b4007efa3dfee026] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=46.9 K 2024-11-12T07:47:40,564 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ead7a74461b4f6a8c59b119c3b3bb4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731397657711 2024-11-12T07:47:40,569 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9268c70da2f94ad5a917547b5f466c04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731397657972 2024-11-12T07:47:40,570 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 57579a172b1347d8be381413928eafde, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731397658696 2024-11-12T07:47:40,572 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 620f91b9c01a4c33b4007efa3dfee026, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397659190 2024-11-12T07:47:40,573 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:40,573 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:40,573 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,574 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/0eb196bd5eb74510a21bfe6866486c22, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f12b0ac12e76435c83af4f09979727fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/4adeb499a63442e88dc9916f219dfd93, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b90d6d80c5c749f98f6ea21cbe5129fa] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=56.0 K 2024-11-12T07:47:40,577 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb196bd5eb74510a21bfe6866486c22, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731397657711 2024-11-12T07:47:40,578 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f12b0ac12e76435c83af4f09979727fb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731397657923 2024-11-12T07:47:40,579 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4adeb499a63442e88dc9916f219dfd93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731397658696 2024-11-12T07:47:40,580 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting b90d6d80c5c749f98f6ea21cbe5129fa, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397659190 2024-11-12T07:47:40,627 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#12 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:40,629 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/381069df1f264d36b4048efee4234b56 is 50, key is test_row_0/A:col10/1731397659842/Put/seqid=0 2024-11-12T07:47:40,637 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#13 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:40,638 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/61e60a18fa5c4e96b5624c5bff0c4581 is 50, key is test_row_0/B:col10/1731397659842/Put/seqid=0 2024-11-12T07:47:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:40,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731397660276, max timestamp=1731397660645Num uniques 1204; 2024-11-12T07:47:40,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731397660276, max timestamp=1731397660645Num uniques 1204; 2024-11-12T07:47:40,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-12T07:47:40,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-12T07:47:40,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731397660276, max timestamp=1731397660645Num uniques 1204; 2024-11-12T07:47:40,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=7050, suffix heap size=153184,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=154 2024-11-12T07:47:40,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=33.54 KB heapSize=86.63 KB 2024-11-12T07:47:40,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:40,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:40,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:40,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:40,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:40,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:40,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741850_1026 (size=12139) 2024-11-12T07:47:40,665 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T07:47:40,666 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T07:47:40,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741849_1025 (size=12139) 2024-11-12T07:47:40,670 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T07:47:40,670 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T07:47:40,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T07:47:40,671 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T07:47:40,673 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T07:47:40,673 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T07:47:40,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/3adffaa7fa344a8b962b0490ac4ce0e6 is 50, key is test_row_0/A:col10/1731397660649/Put/seqid=0 2024-11-12T07:47:40,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,683 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/61e60a18fa5c4e96b5624c5bff0c4581 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/61e60a18fa5c4e96b5624c5bff0c4581 2024-11-12T07:47:40,687 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/381069df1f264d36b4048efee4234b56 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/381069df1f264d36b4048efee4234b56 2024-11-12T07:47:40,709 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into 381069df1f264d36b4048efee4234b56(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:40,709 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:40,712 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=12, startTime=1731397660546; duration=0sec 2024-11-12T07:47:40,713 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:40,713 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:40,713 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:40,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741851_1027 (size=16931) 2024-11-12T07:47:40,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/3adffaa7fa344a8b962b0490ac4ce0e6 2024-11-12T07:47:40,719 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into 61e60a18fa5c4e96b5624c5bff0c4581(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:40,719 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:40,720 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=12, startTime=1731397660550; duration=0sec 2024-11-12T07:47:40,720 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:40,720 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:40,725 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:40,726 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:40,726 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,726 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/23f9f1359e3b460bb8cd5bd071d36e3e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/983cfdadc7f94ce399979f47167e86d0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a00bbb899fa04979a977965aa976a453, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5f15689b442b47c78ba9e2e1e95cd974] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=46.9 K 2024-11-12T07:47:40,728 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23f9f1359e3b460bb8cd5bd071d36e3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731397657711 2024-11-12T07:47:40,729 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 983cfdadc7f94ce399979f47167e86d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731397657972 2024-11-12T07:47:40,731 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00bbb899fa04979a977965aa976a453, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731397658696 2024-11-12T07:47:40,732 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f15689b442b47c78ba9e2e1e95cd974, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397659190 2024-11-12T07:47:40,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731397660681, max timestamp=1731397660729Num uniques 859; 2024-11-12T07:47:40,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731397660681, max timestamp=1731397660729Num uniques 859; 2024-11-12T07:47:40,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T07:47:40,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T07:47:40,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731397660681, max timestamp=1731397660729Num uniques 859; 2024-11-12T07:47:40,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T07:47:40,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e9dbdd7586a64dc99d4d3d07344684f1 is 50, key is test_row_0/B:col10/1731397660649/Put/seqid=0 2024-11-12T07:47:40,777 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#16 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:40,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741852_1028 (size=12151) 2024-11-12T07:47:40,778 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/39013d3400e94273999bac14cc082984 is 50, key is test_row_0/C:col10/1731397659842/Put/seqid=0 2024-11-12T07:47:40,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e9dbdd7586a64dc99d4d3d07344684f1 2024-11-12T07:47:40,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741853_1029 (size=12139) 2024-11-12T07:47:40,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/dbfea798e89a44c193894552ed7a6b18 is 50, key is test_row_0/C:col10/1731397660649/Put/seqid=0 2024-11-12T07:47:40,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397660764, max timestamp=1731397660801Num uniques 1118; 2024-11-12T07:47:40,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-12T07:47:40,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397660764, max timestamp=1731397660801Num uniques 1118; 2024-11-12T07:47:40,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397660764, max timestamp=1731397660801Num uniques 1118; 2024-11-12T07:47:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397720797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-12T07:47:40,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-12T07:47:40,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,821 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/39013d3400e94273999bac14cc082984 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39013d3400e94273999bac14cc082984 2024-11-12T07:47:40,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,839 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into 39013d3400e94273999bac14cc082984(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:40,839 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:40,840 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=12, startTime=1731397660551; duration=0sec 2024-11-12T07:47:40,840 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:40,840 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:40,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741854_1030 (size=12151) 2024-11-12T07:47:40,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/dbfea798e89a44c193894552ed7a6b18 2024-11-12T07:47:40,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/3adffaa7fa344a8b962b0490ac4ce0e6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3adffaa7fa344a8b962b0490ac4ce0e6 2024-11-12T07:47:40,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3adffaa7fa344a8b962b0490ac4ce0e6, entries=250, sequenceid=177, filesize=16.5 K 2024-11-12T07:47:40,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e9dbdd7586a64dc99d4d3d07344684f1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e9dbdd7586a64dc99d4d3d07344684f1 2024-11-12T07:47:40,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731397660858, max timestamp=1731397660885Num uniques 1077; 2024-11-12T07:47:40,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731397660858, max timestamp=1731397660885Num uniques 1077; 2024-11-12T07:47:40,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-12T07:47:40,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-12T07:47:40,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e9dbdd7586a64dc99d4d3d07344684f1, entries=150, sequenceid=177, filesize=11.9 K 2024-11-12T07:47:40,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731397660858, max timestamp=1731397660885Num uniques 1077; 2024-11-12T07:47:40,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T07:47:40,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/dbfea798e89a44c193894552ed7a6b18 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/dbfea798e89a44c193894552ed7a6b18 2024-11-12T07:47:40,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/dbfea798e89a44c193894552ed7a6b18, entries=150, sequenceid=177, filesize=11.9 K 2024-11-12T07:47:40,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=78.27 KB/80150 for 609f0702d0c4b8c7ee5a4c9652b6127b in 264ms, sequenceid=177, compaction requested=false 2024-11-12T07:47:40,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:40,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=91.69 KB heapSize=249.31 KB 2024-11-12T07:47:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:40,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:40,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:40,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:40,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/5112868583354f7bb24b234b3370dc1e is 50, key is test_row_0/A:col10/1731397660927/Put/seqid=0 2024-11-12T07:47:40,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741855_1031 (size=12151) 2024-11-12T07:47:40,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/5112868583354f7bb24b234b3370dc1e 2024-11-12T07:47:40,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397720961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397720962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397720965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397720964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397720981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:40,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:40,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:40,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:40,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:40,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:40,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:41,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e28994dd76f441a8870de1893d2e6bc2 is 50, key is test_row_0/B:col10/1731397660927/Put/seqid=0 2024-11-12T07:47:41,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741856_1032 (size=12151) 2024-11-12T07:47:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T07:47:41,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e28994dd76f441a8870de1893d2e6bc2 2024-11-12T07:47:41,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/f06bd492993e487f8d91397cffdd768f is 50, key is test_row_0/C:col10/1731397660927/Put/seqid=0 2024-11-12T07:47:41,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741857_1033 (size=12151) 2024-11-12T07:47:41,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397721089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397721089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/f06bd492993e487f8d91397cffdd768f 2024-11-12T07:47:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397721089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397721090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397721092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/5112868583354f7bb24b234b3370dc1e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/5112868583354f7bb24b234b3370dc1e 2024-11-12T07:47:41,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/5112868583354f7bb24b234b3370dc1e, entries=150, sequenceid=245, filesize=11.9 K 2024-11-12T07:47:41,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e28994dd76f441a8870de1893d2e6bc2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e28994dd76f441a8870de1893d2e6bc2 2024-11-12T07:47:41,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:41,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:41,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:41,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:41,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e28994dd76f441a8870de1893d2e6bc2, entries=150, sequenceid=245, filesize=11.9 K 2024-11-12T07:47:41,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/f06bd492993e487f8d91397cffdd768f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/f06bd492993e487f8d91397cffdd768f 2024-11-12T07:47:41,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/f06bd492993e487f8d91397cffdd768f, entries=150, sequenceid=245, filesize=11.9 K 2024-11-12T07:47:41,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~105.11 KB/107630, heapSize ~289.11 KB/296048, currentSize=80.51 KB/82440 for 609f0702d0c4b8c7ee5a4c9652b6127b in 250ms, sequenceid=245, compaction requested=true 2024-11-12T07:47:41,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:41,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:41,178 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:41,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:41,179 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:41,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:41,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:41,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:41,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:41,183 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:41,184 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:41,184 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,184 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/381069df1f264d36b4048efee4234b56, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3adffaa7fa344a8b962b0490ac4ce0e6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/5112868583354f7bb24b234b3370dc1e] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=40.3 K 2024-11-12T07:47:41,185 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:41,185 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:41,185 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,185 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/61e60a18fa5c4e96b5624c5bff0c4581, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e9dbdd7586a64dc99d4d3d07344684f1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e28994dd76f441a8870de1893d2e6bc2] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=35.6 K 2024-11-12T07:47:41,186 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 381069df1f264d36b4048efee4234b56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397659190 2024-11-12T07:47:41,187 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e60a18fa5c4e96b5624c5bff0c4581, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397659190 2024-11-12T07:47:41,187 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3adffaa7fa344a8b962b0490ac4ce0e6, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397660276 2024-11-12T07:47:41,188 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e9dbdd7586a64dc99d4d3d07344684f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397660278 2024-11-12T07:47:41,190 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e28994dd76f441a8870de1893d2e6bc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397660921 2024-11-12T07:47:41,190 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5112868583354f7bb24b234b3370dc1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397660921 2024-11-12T07:47:41,211 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:41,212 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/e327853f546a4439a42d08c4ce9c2f30 is 50, key is test_row_0/A:col10/1731397660927/Put/seqid=0 2024-11-12T07:47:41,222 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:41,223 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/fe8990c026994ab093e6b039e71fbbff is 50, key is test_row_0/B:col10/1731397660927/Put/seqid=0 2024-11-12T07:47:41,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741859_1035 (size=12391) 2024-11-12T07:47:41,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741858_1034 (size=12391) 2024-11-12T07:47:41,261 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/e327853f546a4439a42d08c4ce9c2f30 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/e327853f546a4439a42d08c4ce9c2f30 2024-11-12T07:47:41,275 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into e327853f546a4439a42d08c4ce9c2f30(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:41,275 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:41,276 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=13, startTime=1731397661178; duration=0sec 2024-11-12T07:47:41,276 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:41,276 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:41,276 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:41,278 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:41,278 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:41,278 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,279 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39013d3400e94273999bac14cc082984, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/dbfea798e89a44c193894552ed7a6b18, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/f06bd492993e487f8d91397cffdd768f] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=35.6 K 2024-11-12T07:47:41,281 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39013d3400e94273999bac14cc082984, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397659190 2024-11-12T07:47:41,282 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbfea798e89a44c193894552ed7a6b18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397660278 2024-11-12T07:47:41,283 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f06bd492993e487f8d91397cffdd768f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397660921 2024-11-12T07:47:41,305 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:41,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T07:47:41,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,306 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T07:47:41,306 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/d579b4eece7742e483504642cd035903 is 50, key is test_row_0/C:col10/1731397660927/Put/seqid=0 2024-11-12T07:47:41,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:41,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:41,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/225a12a9d0c64119b9d1fc40b5662059 is 50, key is test_row_0/A:col10/1731397660961/Put/seqid=0 2024-11-12T07:47:41,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397721339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397721342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741860_1036 (size=12391) 2024-11-12T07:47:41,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397721345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397721346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397721348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741861_1037 (size=12251) 2024-11-12T07:47:41,368 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/d579b4eece7742e483504642cd035903 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d579b4eece7742e483504642cd035903 2024-11-12T07:47:41,369 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/225a12a9d0c64119b9d1fc40b5662059 2024-11-12T07:47:41,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/f0494dfc5ea64f0b9f10e0b1da98146d is 50, key is test_row_0/B:col10/1731397660961/Put/seqid=0 2024-11-12T07:47:41,389 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into d579b4eece7742e483504642cd035903(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:41,389 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:41,389 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=13, startTime=1731397661179; duration=0sec 2024-11-12T07:47:41,390 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:41,390 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:41,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741862_1038 (size=12251) 2024-11-12T07:47:41,414 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/f0494dfc5ea64f0b9f10e0b1da98146d 2024-11-12T07:47:41,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/26d3630e48f44651aded42c41de58e1f is 50, key is test_row_0/C:col10/1731397660961/Put/seqid=0 2024-11-12T07:47:41,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741863_1039 (size=12251) 2024-11-12T07:47:41,451 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/26d3630e48f44651aded42c41de58e1f 2024-11-12T07:47:41,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397721458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397721461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397721469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397721469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397721469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/225a12a9d0c64119b9d1fc40b5662059 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/225a12a9d0c64119b9d1fc40b5662059 2024-11-12T07:47:41,487 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/225a12a9d0c64119b9d1fc40b5662059, entries=150, sequenceid=261, filesize=12.0 K 2024-11-12T07:47:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/f0494dfc5ea64f0b9f10e0b1da98146d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f0494dfc5ea64f0b9f10e0b1da98146d 2024-11-12T07:47:41,502 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f0494dfc5ea64f0b9f10e0b1da98146d, entries=150, sequenceid=261, filesize=12.0 K 2024-11-12T07:47:41,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/26d3630e48f44651aded42c41de58e1f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/26d3630e48f44651aded42c41de58e1f 2024-11-12T07:47:41,519 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/26d3630e48f44651aded42c41de58e1f, entries=150, sequenceid=261, filesize=12.0 K 2024-11-12T07:47:41,521 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 609f0702d0c4b8c7ee5a4c9652b6127b in 215ms, sequenceid=261, compaction requested=false 2024-11-12T07:47:41,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:41,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-12T07:47:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-12T07:47:41,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T07:47:41,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6270 sec 2024-11-12T07:47:41,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6380 sec 2024-11-12T07:47:41,652 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/fe8990c026994ab093e6b039e71fbbff as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fe8990c026994ab093e6b039e71fbbff 2024-11-12T07:47:41,666 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into fe8990c026994ab093e6b039e71fbbff(size=12.1 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:41,667 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:41,667 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=13, startTime=1731397661178; duration=0sec 2024-11-12T07:47:41,667 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:41,668 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:41,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:47:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:41,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:41,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:41,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:41,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:41,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d0d95fc689154059b285331a0e54d991 is 50, key is test_row_0/A:col10/1731397661339/Put/seqid=0 2024-11-12T07:47:41,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397721697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397721699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397721700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397721702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397721704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741864_1040 (size=12301) 2024-11-12T07:47:41,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d0d95fc689154059b285331a0e54d991 2024-11-12T07:47:41,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/c37f4b1ffcf540039c5f9ecaba8d3a8b is 50, key is test_row_0/B:col10/1731397661339/Put/seqid=0 2024-11-12T07:47:41,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741865_1041 (size=12301) 2024-11-12T07:47:41,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/c37f4b1ffcf540039c5f9ecaba8d3a8b 2024-11-12T07:47:41,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397721808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/9c8a39518ce942e3ad00ae1f94602664 is 50, key is test_row_0/C:col10/1731397661339/Put/seqid=0 2024-11-12T07:47:41,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741866_1042 (size=12301) 2024-11-12T07:47:41,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/9c8a39518ce942e3ad00ae1f94602664 2024-11-12T07:47:41,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397721838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397721839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397721839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397721840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:41,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d0d95fc689154059b285331a0e54d991 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d0d95fc689154059b285331a0e54d991 2024-11-12T07:47:41,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d0d95fc689154059b285331a0e54d991, entries=150, sequenceid=281, filesize=12.0 K 2024-11-12T07:47:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/c37f4b1ffcf540039c5f9ecaba8d3a8b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/c37f4b1ffcf540039c5f9ecaba8d3a8b 2024-11-12T07:47:41,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/c37f4b1ffcf540039c5f9ecaba8d3a8b, entries=150, sequenceid=281, filesize=12.0 K 2024-11-12T07:47:41,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/9c8a39518ce942e3ad00ae1f94602664 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9c8a39518ce942e3ad00ae1f94602664 2024-11-12T07:47:41,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9c8a39518ce942e3ad00ae1f94602664, entries=150, sequenceid=281, filesize=12.0 K 2024-11-12T07:47:41,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 609f0702d0c4b8c7ee5a4c9652b6127b in 223ms, sequenceid=281, compaction requested=true 2024-11-12T07:47:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:41,894 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:41,894 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:41,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:41,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:41,896 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36943 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:41,896 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36943 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:41,896 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:41,896 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:41,896 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,897 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,897 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fe8990c026994ab093e6b039e71fbbff, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f0494dfc5ea64f0b9f10e0b1da98146d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/c37f4b1ffcf540039c5f9ecaba8d3a8b] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=36.1 K 2024-11-12T07:47:41,897 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/e327853f546a4439a42d08c4ce9c2f30, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/225a12a9d0c64119b9d1fc40b5662059, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d0d95fc689154059b285331a0e54d991] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=36.1 K 2024-11-12T07:47:41,897 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting fe8990c026994ab093e6b039e71fbbff, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397660921 2024-11-12T07:47:41,901 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f0494dfc5ea64f0b9f10e0b1da98146d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731397660947 2024-11-12T07:47:41,901 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e327853f546a4439a42d08c4ce9c2f30, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397660921 2024-11-12T07:47:41,902 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c37f4b1ffcf540039c5f9ecaba8d3a8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731397661339 2024-11-12T07:47:41,902 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 225a12a9d0c64119b9d1fc40b5662059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731397660947 2024-11-12T07:47:41,903 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d95fc689154059b285331a0e54d991, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731397661339 2024-11-12T07:47:41,919 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:41,920 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/cb0d342d76fd44f599163141d8173cd0 is 50, key is test_row_0/B:col10/1731397661339/Put/seqid=0 2024-11-12T07:47:41,924 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:41,925 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/da6e9ff3dc554741a1d20267241e8946 is 50, key is test_row_0/A:col10/1731397661339/Put/seqid=0 2024-11-12T07:47:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741867_1043 (size=12643) 2024-11-12T07:47:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741868_1044 (size=12643) 2024-11-12T07:47:41,962 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/cb0d342d76fd44f599163141d8173cd0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cb0d342d76fd44f599163141d8173cd0 2024-11-12T07:47:41,969 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/da6e9ff3dc554741a1d20267241e8946 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/da6e9ff3dc554741a1d20267241e8946 2024-11-12T07:47:41,981 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into cb0d342d76fd44f599163141d8173cd0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:41,981 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:41,981 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=13, startTime=1731397661894; duration=0sec 2024-11-12T07:47:41,982 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:41,982 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:41,982 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:41,985 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36943 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:41,985 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:41,985 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:41,985 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d579b4eece7742e483504642cd035903, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/26d3630e48f44651aded42c41de58e1f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9c8a39518ce942e3ad00ae1f94602664] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=36.1 K 2024-11-12T07:47:41,986 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d579b4eece7742e483504642cd035903, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397660921 2024-11-12T07:47:41,987 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d3630e48f44651aded42c41de58e1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731397660947 2024-11-12T07:47:41,987 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c8a39518ce942e3ad00ae1f94602664, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731397661339 2024-11-12T07:47:41,992 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into da6e9ff3dc554741a1d20267241e8946(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:41,992 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:41,992 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=13, startTime=1731397661894; duration=0sec 2024-11-12T07:47:41,992 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:41,992 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:42,015 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#32 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:42,015 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/882ee1b78bb648adbeb80d743d4b8f1b is 50, key is test_row_0/C:col10/1731397661339/Put/seqid=0 2024-11-12T07:47:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:42,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T07:47:42,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:42,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:42,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:42,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T07:47:42,039 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:42,039 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-12T07:47:42,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741869_1045 (size=12643) 2024-11-12T07:47:42,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d1b7b25243ab49618a23f325209ce9e4 is 50, key is test_row_0/A:col10/1731397662033/Put/seqid=0 2024-11-12T07:47:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T07:47:42,056 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:42,057 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:42,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:42,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741870_1046 (size=12301) 2024-11-12T07:47:42,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d1b7b25243ab49618a23f325209ce9e4 2024-11-12T07:47:42,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/fa050feebdb24cc6b4dea59de14d8b46 is 50, key is test_row_0/B:col10/1731397662033/Put/seqid=0 2024-11-12T07:47:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397722085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397722089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397722096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397722096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397722098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741871_1047 (size=12301) 2024-11-12T07:47:42,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/fa050feebdb24cc6b4dea59de14d8b46 2024-11-12T07:47:42,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6c7a04fe99ec441cba187aaca061daae is 50, key is test_row_0/C:col10/1731397662033/Put/seqid=0 2024-11-12T07:47:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741872_1048 (size=12301) 2024-11-12T07:47:42,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6c7a04fe99ec441cba187aaca061daae 2024-11-12T07:47:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T07:47:42,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d1b7b25243ab49618a23f325209ce9e4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d1b7b25243ab49618a23f325209ce9e4 2024-11-12T07:47:42,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d1b7b25243ab49618a23f325209ce9e4, entries=150, sequenceid=300, filesize=12.0 K 2024-11-12T07:47:42,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/fa050feebdb24cc6b4dea59de14d8b46 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fa050feebdb24cc6b4dea59de14d8b46 2024-11-12T07:47:42,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fa050feebdb24cc6b4dea59de14d8b46, entries=150, sequenceid=300, filesize=12.0 K 2024-11-12T07:47:42,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6c7a04fe99ec441cba187aaca061daae as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c7a04fe99ec441cba187aaca061daae 2024-11-12T07:47:42,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T07:47:42,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:42,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:42,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:42,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:42,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c7a04fe99ec441cba187aaca061daae, entries=150, sequenceid=300, filesize=12.0 K 2024-11-12T07:47:42,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:42,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397722209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397722208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397722209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397722209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397722209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:42,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 609f0702d0c4b8c7ee5a4c9652b6127b in 186ms, sequenceid=300, compaction requested=false 2024-11-12T07:47:42,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T07:47:42,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T07:47:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:42,372 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T07:47:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:42,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/6ede51db708244c5946f2c5fc8c07c40 is 50, key is test_row_0/A:col10/1731397662089/Put/seqid=0 2024-11-12T07:47:42,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741873_1049 (size=12301) 2024-11-12T07:47:42,404 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/6ede51db708244c5946f2c5fc8c07c40 2024-11-12T07:47:42,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e37f79ffc2e1441f86ece3ee6d52109e is 50, key is test_row_0/B:col10/1731397662089/Put/seqid=0 2024-11-12T07:47:42,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:42,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:42,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397722451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741874_1050 (size=12301) 2024-11-12T07:47:42,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397722453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,461 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/882ee1b78bb648adbeb80d743d4b8f1b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/882ee1b78bb648adbeb80d743d4b8f1b 2024-11-12T07:47:42,462 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e37f79ffc2e1441f86ece3ee6d52109e 2024-11-12T07:47:42,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397722458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397722459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397722458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,481 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into 882ee1b78bb648adbeb80d743d4b8f1b(size=12.3 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:42,481 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:42,481 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=13, startTime=1731397661895; duration=0sec 2024-11-12T07:47:42,482 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:42,482 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:42,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6f69df03b14747a39b8974a469e6da41 is 50, key is test_row_0/C:col10/1731397662089/Put/seqid=0 2024-11-12T07:47:42,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741875_1051 (size=12301) 2024-11-12T07:47:42,507 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6f69df03b14747a39b8974a469e6da41 2024-11-12T07:47:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/6ede51db708244c5946f2c5fc8c07c40 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/6ede51db708244c5946f2c5fc8c07c40 2024-11-12T07:47:42,528 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/6ede51db708244c5946f2c5fc8c07c40, entries=150, sequenceid=317, filesize=12.0 K 2024-11-12T07:47:42,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/e37f79ffc2e1441f86ece3ee6d52109e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e37f79ffc2e1441f86ece3ee6d52109e 2024-11-12T07:47:42,545 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e37f79ffc2e1441f86ece3ee6d52109e, entries=150, sequenceid=317, filesize=12.0 K 2024-11-12T07:47:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6f69df03b14747a39b8974a469e6da41 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f69df03b14747a39b8974a469e6da41 2024-11-12T07:47:42,561 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f69df03b14747a39b8974a469e6da41, entries=150, sequenceid=317, filesize=12.0 K 2024-11-12T07:47:42,563 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 609f0702d0c4b8c7ee5a4c9652b6127b in 190ms, sequenceid=317, compaction requested=true 2024-11-12T07:47:42,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:42,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:42,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-12T07:47:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-12T07:47:42,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T07:47:42,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:42,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T07:47:42,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 509 msec 2024-11-12T07:47:42,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:42,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:42,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:42,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 534 msec 2024-11-12T07:47:42,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/f3656d1982e3473eb054b1b48efeff24 is 50, key is test_row_0/A:col10/1731397662570/Put/seqid=0 2024-11-12T07:47:42,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741876_1052 (size=12301) 2024-11-12T07:47:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397722604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397722604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397722608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397722606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397722616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T07:47:42,679 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:42,679 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-12T07:47:42,684 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:42,685 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:42,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:47:42,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397722728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397722729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397722729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397722729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397722729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:47:42,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T07:47:42,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:42,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:42,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:42,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:42,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:42,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397722938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397722938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397722938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397722938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397722948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:42,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:42,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T07:47:42,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:42,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:43,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:47:43,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/f3656d1982e3473eb054b1b48efeff24 2024-11-12T07:47:43,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/093e228f99cb4cdeb3a8c4000cf0cdb6 is 50, key is test_row_0/B:col10/1731397662570/Put/seqid=0 2024-11-12T07:47:43,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741877_1053 (size=12301) 2024-11-12T07:47:43,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/093e228f99cb4cdeb3a8c4000cf0cdb6 2024-11-12T07:47:43,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/530639371ed4456798a343e64cefa327 is 50, key is test_row_0/C:col10/1731397662570/Put/seqid=0 2024-11-12T07:47:43,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741878_1054 (size=12301) 2024-11-12T07:47:43,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T07:47:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397723248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397723248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397723249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397723249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397723258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T07:47:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:47:43,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T07:47:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:43,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/530639371ed4456798a343e64cefa327 2024-11-12T07:47:43,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/f3656d1982e3473eb054b1b48efeff24 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f3656d1982e3473eb054b1b48efeff24 2024-11-12T07:47:43,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f3656d1982e3473eb054b1b48efeff24, entries=150, sequenceid=335, filesize=12.0 K 2024-11-12T07:47:43,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/093e228f99cb4cdeb3a8c4000cf0cdb6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/093e228f99cb4cdeb3a8c4000cf0cdb6 2024-11-12T07:47:43,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/093e228f99cb4cdeb3a8c4000cf0cdb6, entries=150, sequenceid=335, filesize=12.0 K 2024-11-12T07:47:43,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/530639371ed4456798a343e64cefa327 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/530639371ed4456798a343e64cefa327 2024-11-12T07:47:43,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/530639371ed4456798a343e64cefa327, entries=150, sequenceid=335, filesize=12.0 K 2024-11-12T07:47:43,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 609f0702d0c4b8c7ee5a4c9652b6127b in 978ms, sequenceid=335, compaction requested=true 2024-11-12T07:47:43,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:43,551 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:43,551 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:43,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:43,557 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:43,558 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:43,558 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,558 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cb0d342d76fd44f599163141d8173cd0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fa050feebdb24cc6b4dea59de14d8b46, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e37f79ffc2e1441f86ece3ee6d52109e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/093e228f99cb4cdeb3a8c4000cf0cdb6] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=48.4 K 2024-11-12T07:47:43,560 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:43,560 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:43,560 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,560 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/da6e9ff3dc554741a1d20267241e8946, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d1b7b25243ab49618a23f325209ce9e4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/6ede51db708244c5946f2c5fc8c07c40, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f3656d1982e3473eb054b1b48efeff24] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=48.4 K 2024-11-12T07:47:43,561 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0d342d76fd44f599163141d8173cd0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731397661339 2024-11-12T07:47:43,561 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting da6e9ff3dc554741a1d20267241e8946, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731397661339 2024-11-12T07:47:43,561 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting fa050feebdb24cc6b4dea59de14d8b46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731397661696 2024-11-12T07:47:43,561 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b7b25243ab49618a23f325209ce9e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731397661696 2024-11-12T07:47:43,562 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ede51db708244c5946f2c5fc8c07c40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731397662082 2024-11-12T07:47:43,562 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e37f79ffc2e1441f86ece3ee6d52109e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731397662082 2024-11-12T07:47:43,562 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3656d1982e3473eb054b1b48efeff24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731397662434 2024-11-12T07:47:43,563 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 093e228f99cb4cdeb3a8c4000cf0cdb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731397662434 2024-11-12T07:47:43,590 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:43,590 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/7484d288c7cf4ea8a418af1118be44a0 is 50, key is test_row_0/A:col10/1731397662570/Put/seqid=0 2024-11-12T07:47:43,592 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:43,592 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/5ce45c7dc47a4ac2b8351fd5c73d8003 is 50, key is test_row_0/B:col10/1731397662570/Put/seqid=0 2024-11-12T07:47:43,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T07:47:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,620 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T07:47:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:43,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741879_1055 (size=12779) 2024-11-12T07:47:43,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741880_1056 (size=12779) 2024-11-12T07:47:43,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/efbbc729a84a40a2bdb7270cfe89ead8 is 50, key is test_row_0/A:col10/1731397662611/Put/seqid=0 2024-11-12T07:47:43,641 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/7484d288c7cf4ea8a418af1118be44a0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7484d288c7cf4ea8a418af1118be44a0 2024-11-12T07:47:43,643 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/5ce45c7dc47a4ac2b8351fd5c73d8003 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ce45c7dc47a4ac2b8351fd5c73d8003 2024-11-12T07:47:43,658 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into 7484d288c7cf4ea8a418af1118be44a0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:43,658 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:43,658 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into 5ce45c7dc47a4ac2b8351fd5c73d8003(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:43,658 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:43,658 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=12, startTime=1731397663551; duration=0sec 2024-11-12T07:47:43,658 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=12, startTime=1731397663551; duration=0sec 2024-11-12T07:47:43,658 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:43,658 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:43,658 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:43,658 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:43,658 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:43,662 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:43,662 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:43,662 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:43,662 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/882ee1b78bb648adbeb80d743d4b8f1b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c7a04fe99ec441cba187aaca061daae, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f69df03b14747a39b8974a469e6da41, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/530639371ed4456798a343e64cefa327] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=48.4 K 2024-11-12T07:47:43,663 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 882ee1b78bb648adbeb80d743d4b8f1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731397661339 2024-11-12T07:47:43,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741881_1057 (size=12301) 2024-11-12T07:47:43,664 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7a04fe99ec441cba187aaca061daae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731397661696 2024-11-12T07:47:43,665 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/efbbc729a84a40a2bdb7270cfe89ead8 2024-11-12T07:47:43,665 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f69df03b14747a39b8974a469e6da41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731397662082 2024-11-12T07:47:43,666 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 530639371ed4456798a343e64cefa327, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731397662434 2024-11-12T07:47:43,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/d799b5e289154e68a39acce509b82a90 is 50, key is test_row_0/B:col10/1731397662611/Put/seqid=0 2024-11-12T07:47:43,694 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:43,695 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/16ed225157194606a448c3df62cee5be is 50, key is test_row_0/C:col10/1731397662570/Put/seqid=0 2024-11-12T07:47:43,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741882_1058 (size=12301) 2024-11-12T07:47:43,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741883_1059 (size=12779) 2024-11-12T07:47:43,750 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/16ed225157194606a448c3df62cee5be as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/16ed225157194606a448c3df62cee5be 2024-11-12T07:47:43,760 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into 16ed225157194606a448c3df62cee5be(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:43,760 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:43,761 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=12, startTime=1731397663551; duration=0sec 2024-11-12T07:47:43,761 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:43,761 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:43,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:43,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397723793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397723793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397723797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397723800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397723800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:47:43,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397723908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397723908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397723910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397723918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:43,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397723918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,114 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/d799b5e289154e68a39acce509b82a90 2024-11-12T07:47:44,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397724118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397724118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397724118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397724128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397724128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/b83a4d73647847d8b877ae0c415e4ecb is 50, key is test_row_0/C:col10/1731397662611/Put/seqid=0 2024-11-12T07:47:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741884_1060 (size=12301) 2024-11-12T07:47:44,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397724429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397724429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397724429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397724438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397724439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,556 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/b83a4d73647847d8b877ae0c415e4ecb 2024-11-12T07:47:44,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/efbbc729a84a40a2bdb7270cfe89ead8 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/efbbc729a84a40a2bdb7270cfe89ead8 2024-11-12T07:47:44,573 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/efbbc729a84a40a2bdb7270cfe89ead8, entries=150, sequenceid=350, filesize=12.0 K 2024-11-12T07:47:44,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/d799b5e289154e68a39acce509b82a90 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d799b5e289154e68a39acce509b82a90 2024-11-12T07:47:44,583 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d799b5e289154e68a39acce509b82a90, entries=150, sequenceid=350, filesize=12.0 K 2024-11-12T07:47:44,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/b83a4d73647847d8b877ae0c415e4ecb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/b83a4d73647847d8b877ae0c415e4ecb 2024-11-12T07:47:44,596 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/b83a4d73647847d8b877ae0c415e4ecb, entries=150, sequenceid=350, filesize=12.0 K 2024-11-12T07:47:44,602 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 609f0702d0c4b8c7ee5a4c9652b6127b in 982ms, sequenceid=350, compaction requested=false 2024-11-12T07:47:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:44,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-12T07:47:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-12T07:47:44,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T07:47:44,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9190 sec 2024-11-12T07:47:44,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.9280 sec 2024-11-12T07:47:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T07:47:44,848 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:44,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-12T07:47:44,851 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:44,851 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:44,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T07:47:44,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:44,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:47:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:44,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:44,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/b21baa4e9e3249a3bbb0014143bcec41 is 50, key is test_row_0/A:col10/1731397664939/Put/seqid=0 2024-11-12T07:47:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T07:47:44,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397724960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397724960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397724963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397724963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397724968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:44,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741885_1061 (size=14741) 2024-11-12T07:47:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:44,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:44,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/b21baa4e9e3249a3bbb0014143bcec41 2024-11-12T07:47:44,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/63df74214c914943bc8b637fe12d32d1 is 50, key is test_row_0/B:col10/1731397664939/Put/seqid=0 2024-11-12T07:47:45,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741886_1062 (size=12301) 2024-11-12T07:47:45,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T07:47:45,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:45,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397725078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397725078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397725078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397725078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397725088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T07:47:45,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:45,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T07:47:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397725288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397725288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397725288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397725289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397725298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T07:47:45,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:45,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/63df74214c914943bc8b637fe12d32d1 2024-11-12T07:47:45,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/e934615fe9504f3182b42da022a397f0 is 50, key is test_row_0/C:col10/1731397664939/Put/seqid=0 2024-11-12T07:47:45,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741887_1063 (size=12301) 2024-11-12T07:47:45,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T07:47:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:45,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T07:47:45,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397725598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397725598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397725599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397725599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397725618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:45,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T07:47:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T07:47:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:45,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/e934615fe9504f3182b42da022a397f0 2024-11-12T07:47:45,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/b21baa4e9e3249a3bbb0014143bcec41 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b21baa4e9e3249a3bbb0014143bcec41 2024-11-12T07:47:45,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b21baa4e9e3249a3bbb0014143bcec41, entries=200, sequenceid=371, filesize=14.4 K 2024-11-12T07:47:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/63df74214c914943bc8b637fe12d32d1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/63df74214c914943bc8b637fe12d32d1 2024-11-12T07:47:45,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/63df74214c914943bc8b637fe12d32d1, entries=150, sequenceid=371, filesize=12.0 K 2024-11-12T07:47:45,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/e934615fe9504f3182b42da022a397f0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e934615fe9504f3182b42da022a397f0 2024-11-12T07:47:45,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e934615fe9504f3182b42da022a397f0, entries=150, sequenceid=371, filesize=12.0 K 2024-11-12T07:47:45,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 609f0702d0c4b8c7ee5a4c9652b6127b in 923ms, sequenceid=371, compaction requested=true 2024-11-12T07:47:45,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:45,864 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:45,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:45,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:45,864 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:45,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:45,865 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:45,865 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:45,866 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,866 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7484d288c7cf4ea8a418af1118be44a0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/efbbc729a84a40a2bdb7270cfe89ead8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b21baa4e9e3249a3bbb0014143bcec41] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=38.9 K 2024-11-12T07:47:45,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:45,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:45,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:45,866 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:45,866 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:45,867 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,867 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ce45c7dc47a4ac2b8351fd5c73d8003, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d799b5e289154e68a39acce509b82a90, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/63df74214c914943bc8b637fe12d32d1] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=36.5 K 2024-11-12T07:47:45,867 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7484d288c7cf4ea8a418af1118be44a0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731397662434 2024-11-12T07:47:45,867 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ce45c7dc47a4ac2b8351fd5c73d8003, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731397662434 2024-11-12T07:47:45,868 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d799b5e289154e68a39acce509b82a90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731397662603 2024-11-12T07:47:45,869 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbbc729a84a40a2bdb7270cfe89ead8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731397662603 2024-11-12T07:47:45,870 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 63df74214c914943bc8b637fe12d32d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731397663796 2024-11-12T07:47:45,871 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting b21baa4e9e3249a3bbb0014143bcec41, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731397663794 2024-11-12T07:47:45,885 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:45,886 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/81f3000f80b040d18187bebbf383755c is 50, key is test_row_0/B:col10/1731397664939/Put/seqid=0 2024-11-12T07:47:45,893 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:45,894 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/c9934960f6a5481f8cdb9d6a6d9ebbef is 50, key is test_row_0/A:col10/1731397664939/Put/seqid=0 2024-11-12T07:47:45,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741888_1064 (size=12881) 2024-11-12T07:47:45,918 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/81f3000f80b040d18187bebbf383755c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/81f3000f80b040d18187bebbf383755c 2024-11-12T07:47:45,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T07:47:45,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,934 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:47:45,935 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into 81f3000f80b040d18187bebbf383755c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:45,935 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:45,935 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=13, startTime=1731397665864; duration=0sec 2024-11-12T07:47:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:45,935 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:45,935 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:45,935 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:45,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741889_1065 (size=12881) 2024-11-12T07:47:45,937 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:45,937 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:45,938 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:45,938 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/16ed225157194606a448c3df62cee5be, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/b83a4d73647847d8b877ae0c415e4ecb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e934615fe9504f3182b42da022a397f0] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=36.5 K 2024-11-12T07:47:45,938 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 16ed225157194606a448c3df62cee5be, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731397662434 2024-11-12T07:47:45,939 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting b83a4d73647847d8b877ae0c415e4ecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731397662603 2024-11-12T07:47:45,939 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e934615fe9504f3182b42da022a397f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731397663796 2024-11-12T07:47:45,948 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/c9934960f6a5481f8cdb9d6a6d9ebbef as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/c9934960f6a5481f8cdb9d6a6d9ebbef 2024-11-12T07:47:45,958 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into c9934960f6a5481f8cdb9d6a6d9ebbef(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:45,958 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:45,958 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=13, startTime=1731397665864; duration=0sec 2024-11-12T07:47:45,958 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:45,958 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:45,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/ef7371e4684f475592cfb8f74bec19db is 50, key is test_row_0/A:col10/1731397664962/Put/seqid=0 2024-11-12T07:47:45,964 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:45,965 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/fc8c1dd551d04825863118b1b78e4d23 is 50, key is test_row_0/C:col10/1731397664939/Put/seqid=0 2024-11-12T07:47:45,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741891_1067 (size=12301) 2024-11-12T07:47:45,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741890_1066 (size=12881) 2024-11-12T07:47:45,976 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/ef7371e4684f475592cfb8f74bec19db 2024-11-12T07:47:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T07:47:45,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/026f26240b0a411c9d07eaabeded07cf is 50, key is test_row_0/B:col10/1731397664962/Put/seqid=0 2024-11-12T07:47:45,993 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/fc8c1dd551d04825863118b1b78e4d23 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/fc8c1dd551d04825863118b1b78e4d23 2024-11-12T07:47:46,001 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into fc8c1dd551d04825863118b1b78e4d23(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:46,001 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:46,002 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=13, startTime=1731397665866; duration=0sec 2024-11-12T07:47:46,002 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:46,002 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:46,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741892_1068 (size=12301) 2024-11-12T07:47:46,011 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/026f26240b0a411c9d07eaabeded07cf 2024-11-12T07:47:46,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/8d97dd6b2281460bba6abd55a38f308f is 50, key is test_row_0/C:col10/1731397664962/Put/seqid=0 2024-11-12T07:47:46,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741893_1069 (size=12301) 2024-11-12T07:47:46,031 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/8d97dd6b2281460bba6abd55a38f308f 2024-11-12T07:47:46,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/ef7371e4684f475592cfb8f74bec19db as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/ef7371e4684f475592cfb8f74bec19db 2024-11-12T07:47:46,060 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/ef7371e4684f475592cfb8f74bec19db, entries=150, sequenceid=386, filesize=12.0 K 2024-11-12T07:47:46,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/026f26240b0a411c9d07eaabeded07cf as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/026f26240b0a411c9d07eaabeded07cf 2024-11-12T07:47:46,069 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/026f26240b0a411c9d07eaabeded07cf, entries=150, sequenceid=386, filesize=12.0 K 2024-11-12T07:47:46,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/8d97dd6b2281460bba6abd55a38f308f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/8d97dd6b2281460bba6abd55a38f308f 2024-11-12T07:47:46,080 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/8d97dd6b2281460bba6abd55a38f308f, entries=150, sequenceid=386, filesize=12.0 K 2024-11-12T07:47:46,081 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 609f0702d0c4b8c7ee5a4c9652b6127b in 147ms, sequenceid=386, compaction requested=false 2024-11-12T07:47:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:46,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-12T07:47:46,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-12T07:47:46,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T07:47:46,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2310 sec 2024-11-12T07:47:46,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.2390 sec 2024-11-12T07:47:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:46,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:47:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:46,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:46,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:46,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:46,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/455c33e9527c42f68cd33782dd4c848f is 50, key is test_row_0/A:col10/1731397666117/Put/seqid=0 2024-11-12T07:47:46,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741894_1070 (size=12301) 2024-11-12T07:47:46,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/455c33e9527c42f68cd33782dd4c848f 2024-11-12T07:47:46,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731397666143, max timestamp=1731397666164Num uniques 859; 2024-11-12T07:47:46,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T07:47:46,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731397666143, max timestamp=1731397666165Num uniques 859; 2024-11-12T07:47:46,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T07:47:46,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731397666143, max timestamp=1731397666165Num uniques 859; 2024-11-12T07:47:46,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T07:47:46,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6 is 50, key is test_row_0/B:col10/1731397666117/Put/seqid=0 2024-11-12T07:47:46,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741895_1071 (size=12301) 2024-11-12T07:47:46,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397726194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397726202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397726202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397726204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397726204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397726308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397726317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397726318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397726318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397726318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397726518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397726528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397726528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397726528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397726537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6 2024-11-12T07:47:46,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/80addfb44e1b44d9a1f83d5e11909a05 is 50, key is test_row_0/C:col10/1731397666117/Put/seqid=0 2024-11-12T07:47:46,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741896_1072 (size=12301) 2024-11-12T07:47:46,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/80addfb44e1b44d9a1f83d5e11909a05 2024-11-12T07:47:46,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/455c33e9527c42f68cd33782dd4c848f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/455c33e9527c42f68cd33782dd4c848f 2024-11-12T07:47:46,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/455c33e9527c42f68cd33782dd4c848f, entries=150, sequenceid=398, filesize=12.0 K 2024-11-12T07:47:46,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6 2024-11-12T07:47:46,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6, entries=150, sequenceid=398, filesize=12.0 K 2024-11-12T07:47:46,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/80addfb44e1b44d9a1f83d5e11909a05 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/80addfb44e1b44d9a1f83d5e11909a05 2024-11-12T07:47:46,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/80addfb44e1b44d9a1f83d5e11909a05, entries=150, sequenceid=398, filesize=12.0 K 2024-11-12T07:47:46,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 609f0702d0c4b8c7ee5a4c9652b6127b in 533ms, sequenceid=398, compaction requested=true 2024-11-12T07:47:46,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:46,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:46,652 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:46,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:46,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:46,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:46,652 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:46,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:46,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:46,653 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:46,654 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:46,654 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:46,654 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/c9934960f6a5481f8cdb9d6a6d9ebbef, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/ef7371e4684f475592cfb8f74bec19db, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/455c33e9527c42f68cd33782dd4c848f] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=36.6 K 2024-11-12T07:47:46,654 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:46,654 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:46,654 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9934960f6a5481f8cdb9d6a6d9ebbef, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731397663796 2024-11-12T07:47:46,654 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:46,655 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/81f3000f80b040d18187bebbf383755c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/026f26240b0a411c9d07eaabeded07cf, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=36.6 K 2024-11-12T07:47:46,655 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef7371e4684f475592cfb8f74bec19db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731397664943 2024-11-12T07:47:46,655 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f3000f80b040d18187bebbf383755c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731397663796 2024-11-12T07:47:46,656 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 455c33e9527c42f68cd33782dd4c848f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731397666113 2024-11-12T07:47:46,656 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 026f26240b0a411c9d07eaabeded07cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731397664943 2024-11-12T07:47:46,657 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1f9a11ab6e4b9dbce1ec863f8e78b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731397666113 2024-11-12T07:47:46,669 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:46,671 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/952986a175134bee8deb2fab9aba3d48 is 50, key is test_row_0/A:col10/1731397666117/Put/seqid=0 2024-11-12T07:47:46,673 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:46,674 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/f14a90ae9e0e4957b82e43f6f7824256 is 50, key is test_row_0/B:col10/1731397666117/Put/seqid=0 2024-11-12T07:47:46,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741898_1074 (size=12983) 2024-11-12T07:47:46,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741897_1073 (size=12983) 2024-11-12T07:47:46,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:46,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-12T07:47:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:46,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:46,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/89963c140f8044cc8ca76d2301dae6bc is 50, key is test_row_0/A:col10/1731397666201/Put/seqid=0 2024-11-12T07:47:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397726840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397726844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397726846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397726848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397726848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741899_1075 (size=14741) 2024-11-12T07:47:46,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397726958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:46,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T07:47:46,998 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:46,999 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-12T07:47:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T07:47:47,000 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:47,001 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:47,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:47,088 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/f14a90ae9e0e4957b82e43f6f7824256 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f14a90ae9e0e4957b82e43f6f7824256 2024-11-12T07:47:47,097 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into f14a90ae9e0e4957b82e43f6f7824256(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:47,097 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:47,097 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=13, startTime=1731397666652; duration=0sec 2024-11-12T07:47:47,097 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:47,097 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:47,098 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:47,099 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/952986a175134bee8deb2fab9aba3d48 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/952986a175134bee8deb2fab9aba3d48 2024-11-12T07:47:47,101 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:47,102 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:47,102 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,102 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/fc8c1dd551d04825863118b1b78e4d23, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/8d97dd6b2281460bba6abd55a38f308f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/80addfb44e1b44d9a1f83d5e11909a05] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=36.6 K 2024-11-12T07:47:47,106 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting fc8c1dd551d04825863118b1b78e4d23, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731397663796 2024-11-12T07:47:47,107 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d97dd6b2281460bba6abd55a38f308f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731397664943 2024-11-12T07:47:47,107 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 80addfb44e1b44d9a1f83d5e11909a05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731397666113 2024-11-12T07:47:47,108 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into 952986a175134bee8deb2fab9aba3d48(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:47,108 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:47,108 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=13, startTime=1731397666652; duration=0sec 2024-11-12T07:47:47,108 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:47,108 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T07:47:47,116 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:47,116 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/d2b0421c5ec74b3fb5c9e5fce5e1bdce is 50, key is test_row_0/C:col10/1731397666117/Put/seqid=0 2024-11-12T07:47:47,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741900_1076 (size=12983) 2024-11-12T07:47:47,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T07:47:47,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:47,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397727168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397727178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397727178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397727179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/89963c140f8044cc8ca76d2301dae6bc 2024-11-12T07:47:47,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/16533644389c4143aae666e5ea1910cb is 50, key is test_row_0/B:col10/1731397666201/Put/seqid=0 2024-11-12T07:47:47,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741901_1077 (size=12301) 2024-11-12T07:47:47,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T07:47:47,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:47,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T07:47:47,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397727358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T07:47:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397727478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:47,536 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/d2b0421c5ec74b3fb5c9e5fce5e1bdce as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d2b0421c5ec74b3fb5c9e5fce5e1bdce 2024-11-12T07:47:47,544 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into d2b0421c5ec74b3fb5c9e5fce5e1bdce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:47,544 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:47,544 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=13, startTime=1731397666652; duration=0sec 2024-11-12T07:47:47,544 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:47,544 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:47,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T07:47:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T07:47:47,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/16533644389c4143aae666e5ea1910cb 2024-11-12T07:47:47,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/99286b41893148429bbd5f0bccbf1367 is 50, key is test_row_0/C:col10/1731397666201/Put/seqid=0 2024-11-12T07:47:47,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741902_1078 (size=12301) 2024-11-12T07:47:47,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T07:47:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T07:47:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:47,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:47,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397727988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:47,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397727998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397727998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397727998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:48,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T07:47:48,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:48,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:48,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:48,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:48,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:48,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/99286b41893148429bbd5f0bccbf1367 2024-11-12T07:47:48,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/89963c140f8044cc8ca76d2301dae6bc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/89963c140f8044cc8ca76d2301dae6bc 2024-11-12T07:47:48,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/89963c140f8044cc8ca76d2301dae6bc, entries=200, sequenceid=435, filesize=14.4 K 2024-11-12T07:47:48,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/16533644389c4143aae666e5ea1910cb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/16533644389c4143aae666e5ea1910cb 2024-11-12T07:47:48,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/16533644389c4143aae666e5ea1910cb, entries=150, sequenceid=435, filesize=12.0 K 2024-11-12T07:47:48,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/99286b41893148429bbd5f0bccbf1367 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/99286b41893148429bbd5f0bccbf1367 2024-11-12T07:47:48,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/99286b41893148429bbd5f0bccbf1367, entries=150, sequenceid=435, filesize=12.0 K 2024-11-12T07:47:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T07:47:48,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 609f0702d0c4b8c7ee5a4c9652b6127b in 1309ms, sequenceid=435, compaction requested=false 2024-11-12T07:47:48,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:48,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T07:47:48,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:48,229 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T07:47:48,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:48,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:48,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:48,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:48,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:48,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:48,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/8888487f79894487a68406d5194ac70d is 50, key is test_row_0/A:col10/1731397666838/Put/seqid=0 2024-11-12T07:47:48,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741903_1079 (size=12301) 2024-11-12T07:47:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:48,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397668428, max timestamp=1731397668444Num uniques 859; 2024-11-12T07:47:48,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397668428, max timestamp=1731397668444Num uniques 859; 2024-11-12T07:47:48,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397668428, max timestamp=1731397668444Num uniques 859; 2024-11-12T07:47:48,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:47:48,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:47:48,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:47:48,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731397668497, max timestamp=1731397668524Num uniques 1018; 2024-11-12T07:47:48,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T07:47:48,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731397668497, max timestamp=1731397668524Num uniques 1018; 2024-11-12T07:47:48,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T07:47:48,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731397668497, max timestamp=1731397668524Num uniques 1018; 2024-11-12T07:47:48,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-12T07:47:48,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397728631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:48,661 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/8888487f79894487a68406d5194ac70d 2024-11-12T07:47:48,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/6d1ab49d192a4db8bdf85477c5886adb is 50, key is test_row_0/B:col10/1731397666838/Put/seqid=0 2024-11-12T07:47:48,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741904_1080 (size=12301) 2024-11-12T07:47:48,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397728738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397728948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:49,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397728997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:49,009 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 609f0702d0c4b8c7ee5a4c9652b6127b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:47:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397729018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397729018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:49,024 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_0' in 609f0702d0c4b8c7ee5a4c9652b6127b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:47:49,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397729018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:49,025 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 609f0702d0c4b8c7ee5a4c9652b6127b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:47:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:49,025 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 609f0702d0c4b8c7ee5a4c9652b6127b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:47:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:49,076 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/6d1ab49d192a4db8bdf85477c5886adb 2024-11-12T07:47:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/9fba11752c31481abf6ef67736157f02 is 50, key is test_row_0/C:col10/1731397666838/Put/seqid=0 2024-11-12T07:47:49,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741905_1081 (size=12301) 2024-11-12T07:47:49,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T07:47:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397729258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:49,495 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/9fba11752c31481abf6ef67736157f02 2024-11-12T07:47:49,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/8888487f79894487a68406d5194ac70d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8888487f79894487a68406d5194ac70d 2024-11-12T07:47:49,508 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8888487f79894487a68406d5194ac70d, entries=150, sequenceid=446, filesize=12.0 K 2024-11-12T07:47:49,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/6d1ab49d192a4db8bdf85477c5886adb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6d1ab49d192a4db8bdf85477c5886adb 2024-11-12T07:47:49,519 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6d1ab49d192a4db8bdf85477c5886adb, entries=150, sequenceid=446, filesize=12.0 K 2024-11-12T07:47:49,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/9fba11752c31481abf6ef67736157f02 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9fba11752c31481abf6ef67736157f02 2024-11-12T07:47:49,528 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9fba11752c31481abf6ef67736157f02, entries=150, sequenceid=446, filesize=12.0 K 2024-11-12T07:47:49,530 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 609f0702d0c4b8c7ee5a4c9652b6127b in 1301ms, sequenceid=446, compaction requested=true 2024-11-12T07:47:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-12T07:47:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-12T07:47:49,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T07:47:49,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5310 sec 2024-11-12T07:47:49,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5370 sec 2024-11-12T07:47:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:49,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-11-12T07:47:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:47:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:47:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:47:49,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/8d8b3c40e342441cb2617ea7be40b534 is 50, key is test_row_0/A:col10/1731397669770/Put/seqid=0 2024-11-12T07:47:49,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741906_1082 (size=14741) 2024-11-12T07:47:49,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397729793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:49,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397729898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:50,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397730108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:50,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:50,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/8d8b3c40e342441cb2617ea7be40b534 2024-11-12T07:47:50,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/1ec0261a1bfb4ad5b059ff22c2c82c86 is 50, key is test_row_0/B:col10/1731397669770/Put/seqid=0 2024-11-12T07:47:50,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741907_1083 (size=12301) 2024-11-12T07:47:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397730418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:50,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/1ec0261a1bfb4ad5b059ff22c2c82c86 2024-11-12T07:47:50,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/408dc1b8ebad4896be20b5bd4edf4ddc is 50, key is test_row_0/C:col10/1731397669770/Put/seqid=0 2024-11-12T07:47:50,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741908_1084 (size=12301) 2024-11-12T07:47:50,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/408dc1b8ebad4896be20b5bd4edf4ddc 2024-11-12T07:47:50,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/8d8b3c40e342441cb2617ea7be40b534 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8d8b3c40e342441cb2617ea7be40b534 2024-11-12T07:47:50,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8d8b3c40e342441cb2617ea7be40b534, entries=200, sequenceid=501, filesize=14.4 K 2024-11-12T07:47:50,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/1ec0261a1bfb4ad5b059ff22c2c82c86 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/1ec0261a1bfb4ad5b059ff22c2c82c86 2024-11-12T07:47:50,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/1ec0261a1bfb4ad5b059ff22c2c82c86, entries=150, sequenceid=501, filesize=12.0 K 2024-11-12T07:47:50,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/408dc1b8ebad4896be20b5bd4edf4ddc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/408dc1b8ebad4896be20b5bd4edf4ddc 2024-11-12T07:47:50,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/408dc1b8ebad4896be20b5bd4edf4ddc, entries=150, sequenceid=501, filesize=12.0 K 2024-11-12T07:47:50,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=26.84 KB/27480 for 609f0702d0c4b8c7ee5a4c9652b6127b in 876ms, sequenceid=501, compaction requested=true 2024-11-12T07:47:50,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:50,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:50,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:50,648 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:50,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:50,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:50,649 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:50,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:50,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:50,651 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:50,651 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:50,651 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:50,651 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f14a90ae9e0e4957b82e43f6f7824256, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/16533644389c4143aae666e5ea1910cb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6d1ab49d192a4db8bdf85477c5886adb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/1ec0261a1bfb4ad5b059ff22c2c82c86] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=48.7 K 2024-11-12T07:47:50,652 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:50,652 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:50,652 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:50,652 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/952986a175134bee8deb2fab9aba3d48, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/89963c140f8044cc8ca76d2301dae6bc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8888487f79894487a68406d5194ac70d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8d8b3c40e342441cb2617ea7be40b534] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=53.5 K 2024-11-12T07:47:50,652 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f14a90ae9e0e4957b82e43f6f7824256, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731397666113 2024-11-12T07:47:50,653 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 952986a175134bee8deb2fab9aba3d48, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731397666113 2024-11-12T07:47:50,653 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89963c140f8044cc8ca76d2301dae6bc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397666190 2024-11-12T07:47:50,653 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 16533644389c4143aae666e5ea1910cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397666190 2024-11-12T07:47:50,654 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8888487f79894487a68406d5194ac70d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731397666838 2024-11-12T07:47:50,654 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d1ab49d192a4db8bdf85477c5886adb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731397666838 2024-11-12T07:47:50,655 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d8b3c40e342441cb2617ea7be40b534, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731397668606 2024-11-12T07:47:50,655 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec0261a1bfb4ad5b059ff22c2c82c86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731397668606 2024-11-12T07:47:50,669 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:50,670 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/83eea909a2bd4767a4c451d88ddf3c8e is 50, key is test_row_0/A:col10/1731397669770/Put/seqid=0 2024-11-12T07:47:50,672 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:50,673 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/de84b6ec117c40c080a5dd955b40c860 is 50, key is test_row_0/B:col10/1731397669770/Put/seqid=0 2024-11-12T07:47:50,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741909_1085 (size=13119) 2024-11-12T07:47:50,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741910_1086 (size=13119) 2024-11-12T07:47:50,699 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/de84b6ec117c40c080a5dd955b40c860 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/de84b6ec117c40c080a5dd955b40c860 2024-11-12T07:47:50,713 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into de84b6ec117c40c080a5dd955b40c860(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:50,714 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:50,714 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=12, startTime=1731397670649; duration=0sec 2024-11-12T07:47:50,714 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:50,714 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:50,714 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:50,716 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:50,716 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:50,716 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:50,716 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d2b0421c5ec74b3fb5c9e5fce5e1bdce, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/99286b41893148429bbd5f0bccbf1367, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9fba11752c31481abf6ef67736157f02, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/408dc1b8ebad4896be20b5bd4edf4ddc] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=48.7 K 2024-11-12T07:47:50,717 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d2b0421c5ec74b3fb5c9e5fce5e1bdce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731397666113 2024-11-12T07:47:50,717 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 99286b41893148429bbd5f0bccbf1367, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397666190 2024-11-12T07:47:50,717 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fba11752c31481abf6ef67736157f02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731397666838 2024-11-12T07:47:50,718 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 408dc1b8ebad4896be20b5bd4edf4ddc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731397668606 2024-11-12T07:47:50,732 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#74 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:50,733 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/39fad9f8fed24dc3ade49a561cf1c89c is 50, key is test_row_0/C:col10/1731397669770/Put/seqid=0 2024-11-12T07:47:50,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741911_1087 (size=13119) 2024-11-12T07:47:50,755 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/39fad9f8fed24dc3ade49a561cf1c89c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39fad9f8fed24dc3ade49a561cf1c89c 2024-11-12T07:47:50,763 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into 39fad9f8fed24dc3ade49a561cf1c89c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:50,763 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:50,763 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=12, startTime=1731397670649; duration=0sec 2024-11-12T07:47:50,763 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:50,763 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:50,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:47:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:50,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:50,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:50,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:50,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/63cbec73df6b4f4d80936ae17218e04c is 50, key is test_row_0/A:col10/1731397670933/Put/seqid=0 2024-11-12T07:47:50,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741912_1088 (size=12301) 2024-11-12T07:47:51,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397670998, max timestamp=1731397671028Num uniques 859; 2024-11-12T07:47:51,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:47:51,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397670998, max timestamp=1731397671028Num uniques 859; 2024-11-12T07:47:51,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:47:51,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397670998, max timestamp=1731397671028Num uniques 859; 2024-11-12T07:47:51,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:47:51,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397731069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397731069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397731070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397731071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397731072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,092 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/83eea909a2bd4767a4c451d88ddf3c8e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/83eea909a2bd4767a4c451d88ddf3c8e 2024-11-12T07:47:51,106 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into 83eea909a2bd4767a4c451d88ddf3c8e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:51,106 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:51,106 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=12, startTime=1731397670648; duration=0sec 2024-11-12T07:47:51,107 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:51,107 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T07:47:51,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:51,168 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-12T07:47:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T07:47:51,172 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:51,173 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:51,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:51,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397731188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397731188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397731188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397731188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397731188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T07:47:51,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T07:47:51,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:51,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/63cbec73df6b4f4d80936ae17218e04c 2024-11-12T07:47:51,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/8d742d80fb2b4a42ae692cde6cc11d7c is 50, key is test_row_0/B:col10/1731397670933/Put/seqid=0 2024-11-12T07:47:51,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741913_1089 (size=12301) 2024-11-12T07:47:51,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397731397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397731397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397731398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397731398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397731398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T07:47:51,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:51,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T07:47:51,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T07:47:51,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397731707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397731708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397731708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397731708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397731708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:51,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/8d742d80fb2b4a42ae692cde6cc11d7c 2024-11-12T07:47:51,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/ca6aadfed2c5464b8cbd0e83f31d5437 is 50, key is test_row_0/C:col10/1731397670933/Put/seqid=0 2024-11-12T07:47:51,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741914_1090 (size=12301) 2024-11-12T07:47:51,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/ca6aadfed2c5464b8cbd0e83f31d5437 2024-11-12T07:47:51,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T07:47:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:51,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/63cbec73df6b4f4d80936ae17218e04c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/63cbec73df6b4f4d80936ae17218e04c 2024-11-12T07:47:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T07:47:51,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/63cbec73df6b4f4d80936ae17218e04c, entries=150, sequenceid=513, filesize=12.0 K 2024-11-12T07:47:51,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/8d742d80fb2b4a42ae692cde6cc11d7c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/8d742d80fb2b4a42ae692cde6cc11d7c 2024-11-12T07:47:51,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/8d742d80fb2b4a42ae692cde6cc11d7c, entries=150, sequenceid=513, filesize=12.0 K 2024-11-12T07:47:51,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/ca6aadfed2c5464b8cbd0e83f31d5437 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca6aadfed2c5464b8cbd0e83f31d5437 2024-11-12T07:47:51,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca6aadfed2c5464b8cbd0e83f31d5437, entries=150, sequenceid=513, filesize=12.0 K 2024-11-12T07:47:51,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 609f0702d0c4b8c7ee5a4c9652b6127b in 880ms, sequenceid=513, compaction requested=false 2024-11-12T07:47:51,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:51,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T07:47:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:51,940 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T07:47:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/7ab1957a31e3416487134af6069467bb is 50, key is test_row_0/A:col10/1731397671068/Put/seqid=0 2024-11-12T07:47:51,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741915_1091 (size=12301) 2024-11-12T07:47:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:52,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:52,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397732225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397732226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397732226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397732227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397732230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T07:47:52,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397732338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397732338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397732338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397732338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,351 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/7ab1957a31e3416487134af6069467bb 2024-11-12T07:47:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397732348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/d7445b491dc74061be13e4310dccc2a9 is 50, key is test_row_0/B:col10/1731397671068/Put/seqid=0 2024-11-12T07:47:52,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741916_1092 (size=12301) 2024-11-12T07:47:52,365 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/d7445b491dc74061be13e4310dccc2a9 2024-11-12T07:47:52,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/ca815d49f19e44f49a2a46743344591e is 50, key is test_row_0/C:col10/1731397671068/Put/seqid=0 2024-11-12T07:47:52,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741917_1093 (size=12301) 2024-11-12T07:47:52,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397732548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397732548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397732548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397732548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397732558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,781 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/ca815d49f19e44f49a2a46743344591e 2024-11-12T07:47:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/7ab1957a31e3416487134af6069467bb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7ab1957a31e3416487134af6069467bb 2024-11-12T07:47:52,792 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7ab1957a31e3416487134af6069467bb, entries=150, sequenceid=550, filesize=12.0 K 2024-11-12T07:47:52,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/d7445b491dc74061be13e4310dccc2a9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d7445b491dc74061be13e4310dccc2a9 2024-11-12T07:47:52,798 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d7445b491dc74061be13e4310dccc2a9, entries=150, sequenceid=550, filesize=12.0 K 2024-11-12T07:47:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/ca815d49f19e44f49a2a46743344591e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca815d49f19e44f49a2a46743344591e 2024-11-12T07:47:52,804 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca815d49f19e44f49a2a46743344591e, entries=150, sequenceid=550, filesize=12.0 K 2024-11-12T07:47:52,805 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 609f0702d0c4b8c7ee5a4c9652b6127b in 865ms, sequenceid=550, compaction requested=true 2024-11-12T07:47:52,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:52,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:52,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-12T07:47:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-12T07:47:52,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T07:47:52,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-11-12T07:47:52,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6410 sec 2024-11-12T07:47:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:52,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T07:47:52,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:52,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:52,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:52,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:52,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:52,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:52,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/fbf34fd4cfa247719a898b74dc7e3877 is 50, key is test_row_0/A:col10/1731397672229/Put/seqid=0 2024-11-12T07:47:52,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741918_1094 (size=12301) 2024-11-12T07:47:52,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/fbf34fd4cfa247719a898b74dc7e3877 2024-11-12T07:47:52,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/cfe637dc17d3492a84631483e01cc5cd is 50, key is test_row_0/B:col10/1731397672229/Put/seqid=0 2024-11-12T07:47:52,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397732882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731397672874, max timestamp=1731397672891Num uniques 859; 2024-11-12T07:47:52,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T07:47:52,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731397672874, max timestamp=1731397672891Num uniques 859; 2024-11-12T07:47:52,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731397672874, max timestamp=1731397672891Num uniques 859; 2024-11-12T07:47:52,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T07:47:52,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-12T07:47:52,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741919_1095 (size=12301) 2024-11-12T07:47:52,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/cfe637dc17d3492a84631483e01cc5cd 2024-11-12T07:47:52,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/448a5bc02447476a94595a8ec9130194 is 50, key is test_row_0/C:col10/1731397672229/Put/seqid=0 2024-11-12T07:47:52,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741920_1096 (size=12301) 2024-11-12T07:47:52,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397732917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397732918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397732924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397732925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:52,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:52,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397732998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397733028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397733028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397733038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397733038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397733208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397733237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397733248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397733248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397733248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/448a5bc02447476a94595a8ec9130194 2024-11-12T07:47:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T07:47:53,338 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:53,338 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-12T07:47:53,342 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T07:47:53,345 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:53,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:53,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/fbf34fd4cfa247719a898b74dc7e3877 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/fbf34fd4cfa247719a898b74dc7e3877 2024-11-12T07:47:53,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/fbf34fd4cfa247719a898b74dc7e3877, entries=150, sequenceid=562, filesize=12.0 K 2024-11-12T07:47:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/cfe637dc17d3492a84631483e01cc5cd as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cfe637dc17d3492a84631483e01cc5cd 2024-11-12T07:47:53,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cfe637dc17d3492a84631483e01cc5cd, entries=150, sequenceid=562, filesize=12.0 K 2024-11-12T07:47:53,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/448a5bc02447476a94595a8ec9130194 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/448a5bc02447476a94595a8ec9130194 2024-11-12T07:47:53,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/448a5bc02447476a94595a8ec9130194, entries=150, sequenceid=562, filesize=12.0 K 2024-11-12T07:47:53,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 609f0702d0c4b8c7ee5a4c9652b6127b in 508ms, sequenceid=562, compaction requested=true 2024-11-12T07:47:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:53,369 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:53,369 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:53,370 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:53,371 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:53,371 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:53,371 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/de84b6ec117c40c080a5dd955b40c860, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/8d742d80fb2b4a42ae692cde6cc11d7c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d7445b491dc74061be13e4310dccc2a9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cfe637dc17d3492a84631483e01cc5cd] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=48.8 K 2024-11-12T07:47:53,371 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:53,371 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:53,371 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:53,371 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/83eea909a2bd4767a4c451d88ddf3c8e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/63cbec73df6b4f4d80936ae17218e04c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7ab1957a31e3416487134af6069467bb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/fbf34fd4cfa247719a898b74dc7e3877] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=48.8 K 2024-11-12T07:47:53,372 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting de84b6ec117c40c080a5dd955b40c860, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731397668606 2024-11-12T07:47:53,372 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83eea909a2bd4767a4c451d88ddf3c8e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731397668606 2024-11-12T07:47:53,372 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63cbec73df6b4f4d80936ae17218e04c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731397669791 2024-11-12T07:47:53,372 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d742d80fb2b4a42ae692cde6cc11d7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731397669791 2024-11-12T07:47:53,373 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d7445b491dc74061be13e4310dccc2a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731397671064 2024-11-12T07:47:53,373 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ab1957a31e3416487134af6069467bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731397671064 2024-11-12T07:47:53,373 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting cfe637dc17d3492a84631483e01cc5cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731397672229 2024-11-12T07:47:53,373 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbf34fd4cfa247719a898b74dc7e3877, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731397672229 2024-11-12T07:47:53,387 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:53,387 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/8c7a5f10b62148a6991e3a8fd127e0c2 is 50, key is test_row_0/A:col10/1731397672229/Put/seqid=0 2024-11-12T07:47:53,390 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:53,391 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/4b0e5affbcf34d6ea44bf42b235853e9 is 50, key is test_row_0/B:col10/1731397672229/Put/seqid=0 2024-11-12T07:47:53,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741921_1097 (size=13255) 2024-11-12T07:47:53,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741922_1098 (size=13255) 2024-11-12T07:47:53,422 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/4b0e5affbcf34d6ea44bf42b235853e9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/4b0e5affbcf34d6ea44bf42b235853e9 2024-11-12T07:47:53,429 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into 4b0e5affbcf34d6ea44bf42b235853e9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:53,429 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:53,430 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=12, startTime=1731397673369; duration=0sec 2024-11-12T07:47:53,430 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:53,432 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:53,432 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:53,434 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:53,434 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:53,434 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:53,434 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39fad9f8fed24dc3ade49a561cf1c89c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca6aadfed2c5464b8cbd0e83f31d5437, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca815d49f19e44f49a2a46743344591e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/448a5bc02447476a94595a8ec9130194] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=48.8 K 2024-11-12T07:47:53,434 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 39fad9f8fed24dc3ade49a561cf1c89c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731397668606 2024-11-12T07:47:53,435 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ca6aadfed2c5464b8cbd0e83f31d5437, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731397669791 2024-11-12T07:47:53,435 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ca815d49f19e44f49a2a46743344591e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731397671064 2024-11-12T07:47:53,435 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 448a5bc02447476a94595a8ec9130194, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731397672229 2024-11-12T07:47:53,446 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:53,447 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/e22619d839fa4024afbed99313a96c95 is 50, key is test_row_0/C:col10/1731397672229/Put/seqid=0 2024-11-12T07:47:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T07:47:53,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741923_1099 (size=13255) 2024-11-12T07:47:53,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-12T07:47:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:53,498 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=127.47 KB heapSize=351 KB 2024-11-12T07:47:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d828b6a7fd3c45b9a84b869d6afdb327 is 50, key is test_row_0/A:col10/1731397672916/Put/seqid=0 2024-11-12T07:47:53,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741924_1100 (size=12301) 2024-11-12T07:47:53,518 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d828b6a7fd3c45b9a84b869d6afdb327 2024-11-12T07:47:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/80dbe372f14349879c10d24d76192d40 is 50, key is test_row_0/B:col10/1731397672916/Put/seqid=0 2024-11-12T07:47:53,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397733555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741925_1101 (size=12301) 2024-11-12T07:47:53,563 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/80dbe372f14349879c10d24d76192d40 2024-11-12T07:47:53,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397733558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397733558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397733558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397733558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/a2d60b50be8c4a66bc3aa7e931518d17 is 50, key is test_row_0/C:col10/1731397672916/Put/seqid=0 2024-11-12T07:47:53,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741926_1102 (size=12301) 2024-11-12T07:47:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T07:47:53,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397733668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397733678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,813 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/8c7a5f10b62148a6991e3a8fd127e0c2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8c7a5f10b62148a6991e3a8fd127e0c2 2024-11-12T07:47:53,846 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into 8c7a5f10b62148a6991e3a8fd127e0c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:53,846 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:53,846 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=12, startTime=1731397673369; duration=0sec 2024-11-12T07:47:53,847 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:53,847 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:53,876 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/e22619d839fa4024afbed99313a96c95 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e22619d839fa4024afbed99313a96c95 2024-11-12T07:47:53,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397733877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,882 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into e22619d839fa4024afbed99313a96c95(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:53,882 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:53,882 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=12, startTime=1731397673369; duration=0sec 2024-11-12T07:47:53,882 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:53,882 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:53,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397733888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T07:47:53,989 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=596 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/a2d60b50be8c4a66bc3aa7e931518d17 2024-11-12T07:47:53,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d828b6a7fd3c45b9a84b869d6afdb327 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d828b6a7fd3c45b9a84b869d6afdb327 2024-11-12T07:47:54,003 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d828b6a7fd3c45b9a84b869d6afdb327, entries=150, sequenceid=596, filesize=12.0 K 2024-11-12T07:47:54,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/80dbe372f14349879c10d24d76192d40 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/80dbe372f14349879c10d24d76192d40 2024-11-12T07:47:54,014 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/80dbe372f14349879c10d24d76192d40, entries=150, sequenceid=596, filesize=12.0 K 2024-11-12T07:47:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/a2d60b50be8c4a66bc3aa7e931518d17 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a2d60b50be8c4a66bc3aa7e931518d17 2024-11-12T07:47:54,024 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a2d60b50be8c4a66bc3aa7e931518d17, entries=150, sequenceid=596, filesize=12.0 K 2024-11-12T07:47:54,025 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~350.95 KB/359376, currentSize=67.09 KB/68700 for 609f0702d0c4b8c7ee5a4c9652b6127b in 527ms, sequenceid=596, compaction requested=false 2024-11-12T07:47:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-12T07:47:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-12T07:47:54,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T07:47:54,030 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 683 msec 2024-11-12T07:47:54,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 691 msec 2024-11-12T07:47:54,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:47:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:54,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:54,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:54,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/9a23524f3d2b428da946c24069319f96 is 50, key is test_row_0/A:col10/1731397674079/Put/seqid=0 2024-11-12T07:47:54,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397734107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397734109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397734111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741927_1103 (size=9857) 2024-11-12T07:47:54,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/9a23524f3d2b428da946c24069319f96 2024-11-12T07:47:54,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/41fe9b9bac65439e858d10a1d6299d92 is 50, key is test_row_0/B:col10/1731397674079/Put/seqid=0 2024-11-12T07:47:54,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741928_1104 (size=9857) 2024-11-12T07:47:54,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/41fe9b9bac65439e858d10a1d6299d92 2024-11-12T07:47:54,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/1c105333abeb42d3967c50d8e95e02ef is 50, key is test_row_0/C:col10/1731397674079/Put/seqid=0 2024-11-12T07:47:54,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741929_1105 (size=9857) 2024-11-12T07:47:54,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/1c105333abeb42d3967c50d8e95e02ef 2024-11-12T07:47:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/9a23524f3d2b428da946c24069319f96 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9a23524f3d2b428da946c24069319f96 2024-11-12T07:47:54,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9a23524f3d2b428da946c24069319f96, entries=100, sequenceid=613, filesize=9.6 K 2024-11-12T07:47:54,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/41fe9b9bac65439e858d10a1d6299d92 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/41fe9b9bac65439e858d10a1d6299d92 2024-11-12T07:47:54,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/41fe9b9bac65439e858d10a1d6299d92, entries=100, sequenceid=613, filesize=9.6 K 2024-11-12T07:47:54,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/1c105333abeb42d3967c50d8e95e02ef as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/1c105333abeb42d3967c50d8e95e02ef 2024-11-12T07:47:54,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/1c105333abeb42d3967c50d8e95e02ef, entries=100, sequenceid=613, filesize=9.6 K 2024-11-12T07:47:54,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 609f0702d0c4b8c7ee5a4c9652b6127b in 117ms, sequenceid=613, compaction requested=true 2024-11-12T07:47:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:54,198 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:54,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:54,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:54,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:54,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:54,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T07:47:54,199 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:54,200 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:54,200 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:54,200 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,201 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8c7a5f10b62148a6991e3a8fd127e0c2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d828b6a7fd3c45b9a84b869d6afdb327, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9a23524f3d2b428da946c24069319f96] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=34.6 K 2024-11-12T07:47:54,201 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7a5f10b62148a6991e3a8fd127e0c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731397672229 2024-11-12T07:47:54,202 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:54,202 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:54,202 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,202 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/4b0e5affbcf34d6ea44bf42b235853e9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/80dbe372f14349879c10d24d76192d40, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/41fe9b9bac65439e858d10a1d6299d92] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=34.6 K 2024-11-12T07:47:54,203 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d828b6a7fd3c45b9a84b869d6afdb327, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731397672916 2024-11-12T07:47:54,203 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b0e5affbcf34d6ea44bf42b235853e9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731397672229 2024-11-12T07:47:54,203 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a23524f3d2b428da946c24069319f96, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1731397674079 2024-11-12T07:47:54,204 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80dbe372f14349879c10d24d76192d40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731397672916 2024-11-12T07:47:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:54,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:47:54,206 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41fe9b9bac65439e858d10a1d6299d92, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1731397674079 2024-11-12T07:47:54,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:54,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:54,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:54,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:54,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:54,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:54,224 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:54,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/217891ea93144abb9189f05bfdc64445 is 50, key is test_row_0/A:col10/1731397674204/Put/seqid=0 2024-11-12T07:47:54,224 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/207680040e764953886476e3ad6593fd is 50, key is test_row_0/B:col10/1731397674079/Put/seqid=0 2024-11-12T07:47:54,229 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:54,229 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d78b671c38ec4267b011e147764e1585 is 50, key is test_row_0/A:col10/1731397674079/Put/seqid=0 2024-11-12T07:47:54,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397734229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397734229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397734234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741931_1107 (size=13357) 2024-11-12T07:47:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741932_1108 (size=13357) 2024-11-12T07:47:54,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397734238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397734239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741930_1106 (size=14741) 2024-11-12T07:47:54,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/217891ea93144abb9189f05bfdc64445 2024-11-12T07:47:54,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/fcc0ae5da9bc4885b6e760be7c9fe0dd is 50, key is test_row_0/B:col10/1731397674204/Put/seqid=0 2024-11-12T07:47:54,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741933_1109 (size=12301) 2024-11-12T07:47:54,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397734347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397734348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397734348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397734348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397734348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T07:47:54,478 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:54,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-12T07:47:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T07:47:54,481 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:54,482 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:54,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:54,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397734558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397734558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397734558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397734558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397734558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T07:47:54,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T07:47:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:54,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:54,647 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/d78b671c38ec4267b011e147764e1585 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d78b671c38ec4267b011e147764e1585 2024-11-12T07:47:54,650 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/207680040e764953886476e3ad6593fd as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/207680040e764953886476e3ad6593fd 2024-11-12T07:47:54,657 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into d78b671c38ec4267b011e147764e1585(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:54,657 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into 207680040e764953886476e3ad6593fd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:54,657 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:54,657 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:54,657 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=13, startTime=1731397674198; duration=0sec 2024-11-12T07:47:54,657 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=13, startTime=1731397674199; duration=0sec 2024-11-12T07:47:54,657 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:54,657 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:54,657 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:54,657 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:54,657 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:47:54,658 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:47:54,659 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:54,659 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,659 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e22619d839fa4024afbed99313a96c95, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a2d60b50be8c4a66bc3aa7e931518d17, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/1c105333abeb42d3967c50d8e95e02ef] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=34.6 K 2024-11-12T07:47:54,660 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22619d839fa4024afbed99313a96c95, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731397672229 2024-11-12T07:47:54,660 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d60b50be8c4a66bc3aa7e931518d17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=596, earliestPutTs=1731397672916 2024-11-12T07:47:54,661 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c105333abeb42d3967c50d8e95e02ef, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1731397674079 2024-11-12T07:47:54,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/fcc0ae5da9bc4885b6e760be7c9fe0dd 2024-11-12T07:47:54,670 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:54,671 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6f563ee1982d4b4db0905866428e9d8e is 50, key is test_row_0/C:col10/1731397674079/Put/seqid=0 2024-11-12T07:47:54,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741934_1110 (size=13357) 2024-11-12T07:47:54,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/d28f5cb4dee543d4a917f947cbcf07d7 is 50, key is test_row_0/C:col10/1731397674204/Put/seqid=0 2024-11-12T07:47:54,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741935_1111 (size=12301) 2024-11-12T07:47:54,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T07:47:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T07:47:54,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397734868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397734869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397734869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397734869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397734869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:54,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T07:47:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:54,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:55,099 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6f563ee1982d4b4db0905866428e9d8e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f563ee1982d4b4db0905866428e9d8e 2024-11-12T07:47:55,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T07:47:55,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:55,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:55,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:55,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:55,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:55,107 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into 6f563ee1982d4b4db0905866428e9d8e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:55,107 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:55,108 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=13, startTime=1731397674199; duration=0sec 2024-11-12T07:47:55,108 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:55,108 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T07:47:55,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/d28f5cb4dee543d4a917f947cbcf07d7 2024-11-12T07:47:55,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/217891ea93144abb9189f05bfdc64445 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/217891ea93144abb9189f05bfdc64445 2024-11-12T07:47:55,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/217891ea93144abb9189f05bfdc64445, entries=200, sequenceid=632, filesize=14.4 K 2024-11-12T07:47:55,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/fcc0ae5da9bc4885b6e760be7c9fe0dd as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fcc0ae5da9bc4885b6e760be7c9fe0dd 2024-11-12T07:47:55,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fcc0ae5da9bc4885b6e760be7c9fe0dd, entries=150, sequenceid=632, filesize=12.0 K 2024-11-12T07:47:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/d28f5cb4dee543d4a917f947cbcf07d7 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d28f5cb4dee543d4a917f947cbcf07d7 2024-11-12T07:47:55,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d28f5cb4dee543d4a917f947cbcf07d7, entries=150, sequenceid=632, filesize=12.0 K 2024-11-12T07:47:55,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 609f0702d0c4b8c7ee5a4c9652b6127b in 943ms, sequenceid=632, compaction requested=false 2024-11-12T07:47:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:55,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-12T07:47:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:55,255 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:47:55,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:55,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:55,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:55,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:55,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:55,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/834f18957a15460f877fe343059e1ded is 50, key is test_row_0/A:col10/1731397674234/Put/seqid=0 2024-11-12T07:47:55,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741936_1112 (size=12301) 2024-11-12T07:47:55,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:55,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397735396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397735400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397735404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397735405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397735405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397735508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397735508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397735518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397735518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397735518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T07:47:55,685 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/834f18957a15460f877fe343059e1ded 2024-11-12T07:47:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/01501e24f65a42928112cfa0efab9c84 is 50, key is test_row_0/B:col10/1731397674234/Put/seqid=0 2024-11-12T07:47:55,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741937_1113 (size=12301) 2024-11-12T07:47:55,705 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/01501e24f65a42928112cfa0efab9c84 2024-11-12T07:47:55,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6da4a74679464637885d73dc45fb647e is 50, key is test_row_0/C:col10/1731397674234/Put/seqid=0 2024-11-12T07:47:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397735717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397735717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397735729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397735729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397735730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:55,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741938_1114 (size=12301) 2024-11-12T07:47:55,739 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=649 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6da4a74679464637885d73dc45fb647e 2024-11-12T07:47:55,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/834f18957a15460f877fe343059e1ded as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/834f18957a15460f877fe343059e1ded 2024-11-12T07:47:55,753 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/834f18957a15460f877fe343059e1ded, entries=150, sequenceid=649, filesize=12.0 K 2024-11-12T07:47:55,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/01501e24f65a42928112cfa0efab9c84 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/01501e24f65a42928112cfa0efab9c84 2024-11-12T07:47:55,758 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/01501e24f65a42928112cfa0efab9c84, entries=150, sequenceid=649, filesize=12.0 K 2024-11-12T07:47:55,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6da4a74679464637885d73dc45fb647e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6da4a74679464637885d73dc45fb647e 2024-11-12T07:47:55,767 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6da4a74679464637885d73dc45fb647e, entries=150, sequenceid=649, filesize=12.0 K 2024-11-12T07:47:55,768 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 609f0702d0c4b8c7ee5a4c9652b6127b in 513ms, sequenceid=649, compaction requested=true 2024-11-12T07:47:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:55,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-12T07:47:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-12T07:47:55,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-12T07:47:55,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2870 sec 2024-11-12T07:47:55,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.2930 sec 2024-11-12T07:47:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:56,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T07:47:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:56,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:47:56,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/9e929d49391042f9aaa10622ab108200 is 50, key is test_row_0/A:col10/1731397676038/Put/seqid=0 2024-11-12T07:47:56,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741939_1115 (size=12301) 2024-11-12T07:47:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397736048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397736051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/9e929d49391042f9aaa10622ab108200 2024-11-12T07:47:56,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397736052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397736048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397736054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/78bf0a03c33f4877956862c827f55e01 is 50, key is test_row_0/B:col10/1731397676038/Put/seqid=0 2024-11-12T07:47:56,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741940_1116 (size=12301) 2024-11-12T07:47:56,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/78bf0a03c33f4877956862c827f55e01 2024-11-12T07:47:56,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/5868c203b7174de2a8b1b0edaa11aca6 is 50, key is test_row_0/C:col10/1731397676038/Put/seqid=0 2024-11-12T07:47:56,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741941_1117 (size=12301) 2024-11-12T07:47:56,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397736158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397736158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397736158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397736168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397736168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397736368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397736368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397736368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397736378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397736378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=670 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/5868c203b7174de2a8b1b0edaa11aca6 2024-11-12T07:47:56,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/9e929d49391042f9aaa10622ab108200 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9e929d49391042f9aaa10622ab108200 2024-11-12T07:47:56,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9e929d49391042f9aaa10622ab108200, entries=150, sequenceid=670, filesize=12.0 K 2024-11-12T07:47:56,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/78bf0a03c33f4877956862c827f55e01 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/78bf0a03c33f4877956862c827f55e01 2024-11-12T07:47:56,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/78bf0a03c33f4877956862c827f55e01, entries=150, sequenceid=670, filesize=12.0 K 2024-11-12T07:47:56,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/5868c203b7174de2a8b1b0edaa11aca6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5868c203b7174de2a8b1b0edaa11aca6 2024-11-12T07:47:56,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5868c203b7174de2a8b1b0edaa11aca6, entries=150, sequenceid=670, filesize=12.0 K 2024-11-12T07:47:56,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 609f0702d0c4b8c7ee5a4c9652b6127b in 486ms, sequenceid=670, compaction requested=true 2024-11-12T07:47:56,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:56,526 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:56,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:56,527 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:56,531 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:56,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:56,531 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/A is initiating minor compaction (all files) 2024-11-12T07:47:56,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:56,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:56,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:56,531 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/A in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:56,531 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d78b671c38ec4267b011e147764e1585, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/217891ea93144abb9189f05bfdc64445, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/834f18957a15460f877fe343059e1ded, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9e929d49391042f9aaa10622ab108200] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=51.5 K 2024-11-12T07:47:56,532 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:56,532 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/B is initiating minor compaction (all files) 2024-11-12T07:47:56,532 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/B in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:56,532 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78b671c38ec4267b011e147764e1585, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1731397672922 2024-11-12T07:47:56,532 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/207680040e764953886476e3ad6593fd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fcc0ae5da9bc4885b6e760be7c9fe0dd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/01501e24f65a42928112cfa0efab9c84, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/78bf0a03c33f4877956862c827f55e01] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=49.1 K 2024-11-12T07:47:56,533 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 217891ea93144abb9189f05bfdc64445, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1731397674105 2024-11-12T07:47:56,533 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 207680040e764953886476e3ad6593fd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1731397672922 2024-11-12T07:47:56,533 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting fcc0ae5da9bc4885b6e760be7c9fe0dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1731397674105 2024-11-12T07:47:56,533 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834f18957a15460f877fe343059e1ded, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731397674233 2024-11-12T07:47:56,533 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 01501e24f65a42928112cfa0efab9c84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731397674233 2024-11-12T07:47:56,533 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e929d49391042f9aaa10622ab108200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=670, earliestPutTs=1731397675395 2024-11-12T07:47:56,534 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bf0a03c33f4877956862c827f55e01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=670, earliestPutTs=1731397675395 2024-11-12T07:47:56,549 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#A#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:56,549 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:56,550 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/4ffaeb5daf0643f7917531b082825f30 is 50, key is test_row_0/B:col10/1731397676038/Put/seqid=0 2024-11-12T07:47:56,550 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/21c3ddade7e843df96ab5a27ff23ab0c is 50, key is test_row_0/A:col10/1731397676038/Put/seqid=0 2024-11-12T07:47:56,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741942_1118 (size=13493) 2024-11-12T07:47:56,597 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/21c3ddade7e843df96ab5a27ff23ab0c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/21c3ddade7e843df96ab5a27ff23ab0c 2024-11-12T07:47:56,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741943_1119 (size=13493) 2024-11-12T07:47:56,606 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/A of 609f0702d0c4b8c7ee5a4c9652b6127b into 21c3ddade7e843df96ab5a27ff23ab0c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:56,606 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:56,607 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/A, priority=12, startTime=1731397676526; duration=0sec 2024-11-12T07:47:56,607 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:56,607 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:56,607 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:47:56,609 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:47:56,610 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 609f0702d0c4b8c7ee5a4c9652b6127b/C is initiating minor compaction (all files) 2024-11-12T07:47:56,610 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 609f0702d0c4b8c7ee5a4c9652b6127b/C in TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:56,610 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f563ee1982d4b4db0905866428e9d8e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d28f5cb4dee543d4a917f947cbcf07d7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6da4a74679464637885d73dc45fb647e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5868c203b7174de2a8b1b0edaa11aca6] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp, totalSize=49.1 K 2024-11-12T07:47:56,615 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/4ffaeb5daf0643f7917531b082825f30 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/4ffaeb5daf0643f7917531b082825f30 2024-11-12T07:47:56,617 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f563ee1982d4b4db0905866428e9d8e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1731397672922 2024-11-12T07:47:56,618 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28f5cb4dee543d4a917f947cbcf07d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=632, earliestPutTs=1731397674105 2024-11-12T07:47:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-12T07:47:56,628 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:56,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:47:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-12T07:47:56,631 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:47:56,631 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da4a74679464637885d73dc45fb647e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=649, earliestPutTs=1731397674233 2024-11-12T07:47:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:47:56,632 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5868c203b7174de2a8b1b0edaa11aca6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=670, earliestPutTs=1731397675395 2024-11-12T07:47:56,632 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:47:56,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:47:56,637 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/B of 609f0702d0c4b8c7ee5a4c9652b6127b into 4ffaeb5daf0643f7917531b082825f30(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:56,637 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:56,637 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/B, priority=12, startTime=1731397676527; duration=0sec 2024-11-12T07:47:56,637 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:56,637 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:56,650 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 609f0702d0c4b8c7ee5a4c9652b6127b#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:47:56,651 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/0f33da7b9e344e14a4baff86ed5955af is 50, key is test_row_0/C:col10/1731397676038/Put/seqid=0 2024-11-12T07:47:56,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741944_1120 (size=13493) 2024-11-12T07:47:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:56,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T07:47:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:56,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/829612420aca40e4b27185fee27f4585 is 50, key is test_row_0/A:col10/1731397676049/Put/seqid=0 2024-11-12T07:47:56,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741945_1121 (size=17181) 2024-11-12T07:47:56,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=686 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/829612420aca40e4b27185fee27f4585 2024-11-12T07:47:56,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/b885eb8b2d2e4173a1cfd17ccda92da8 is 50, key is test_row_0/B:col10/1731397676049/Put/seqid=0 2024-11-12T07:47:56,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397736705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741946_1122 (size=12301) 2024-11-12T07:47:56,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397736705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397736707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397736716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397736716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:47:56,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-12T07:47:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:56,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:56,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397736828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397736828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397736828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397736829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397736829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:56,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:56,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-12T07:47:56,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:56,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:56,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:56,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:56,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:47:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397737037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397737038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397737038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397737038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397737038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,080 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/0f33da7b9e344e14a4baff86ed5955af as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/0f33da7b9e344e14a4baff86ed5955af 2024-11-12T07:47:57,089 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 609f0702d0c4b8c7ee5a4c9652b6127b/C of 609f0702d0c4b8c7ee5a4c9652b6127b into 0f33da7b9e344e14a4baff86ed5955af(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:47:57,089 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:57,089 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., storeName=609f0702d0c4b8c7ee5a4c9652b6127b/C, priority=12, startTime=1731397676531; duration=0sec 2024-11-12T07:47:57,089 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:57,090 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:57,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-12T07:47:57,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:57,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:57,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:57,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:57,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:47:57,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=686 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/b885eb8b2d2e4173a1cfd17ccda92da8 2024-11-12T07:47:57,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/0de975c60d9a4a81acb63fc2cf23ebcb is 50, key is test_row_0/C:col10/1731397676049/Put/seqid=0 2024-11-12T07:47:57,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741947_1123 (size=12301) 2024-11-12T07:47:57,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=686 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/0de975c60d9a4a81acb63fc2cf23ebcb 2024-11-12T07:47:57,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/829612420aca40e4b27185fee27f4585 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/829612420aca40e4b27185fee27f4585 2024-11-12T07:47:57,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/829612420aca40e4b27185fee27f4585, entries=250, sequenceid=686, filesize=16.8 K 2024-11-12T07:47:57,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/b885eb8b2d2e4173a1cfd17ccda92da8 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/b885eb8b2d2e4173a1cfd17ccda92da8 2024-11-12T07:47:57,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/b885eb8b2d2e4173a1cfd17ccda92da8, entries=150, sequenceid=686, filesize=12.0 K 2024-11-12T07:47:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/0de975c60d9a4a81acb63fc2cf23ebcb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/0de975c60d9a4a81acb63fc2cf23ebcb 2024-11-12T07:47:57,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/0de975c60d9a4a81acb63fc2cf23ebcb, entries=150, sequenceid=686, filesize=12.0 K 2024-11-12T07:47:57,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 609f0702d0c4b8c7ee5a4c9652b6127b in 501ms, sequenceid=686, compaction requested=false 2024-11-12T07:47:57,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:57,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-12T07:47:57,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:57,246 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:47:57,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:57,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:57,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:57,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/3fd7510f623d4a158a9dde3f1b57fd62 is 50, key is test_row_0/A:col10/1731397676711/Put/seqid=0 2024-11-12T07:47:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:47:57,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741948_1124 (size=12301) 2024-11-12T07:47:57,286 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/3fd7510f623d4a158a9dde3f1b57fd62 2024-11-12T07:47:57,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/554511800eeb42b390909cb750f3db0e is 50, key is test_row_0/B:col10/1731397676711/Put/seqid=0 2024-11-12T07:47:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741949_1125 (size=12301) 2024-11-12T07:47:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:57,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. as already flushing 2024-11-12T07:47:57,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397737364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397737365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397737366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397737367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397737374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397737478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397737489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397737489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397737489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397737489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397737688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397737697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397737697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397737698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397737698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:57,714 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-12T07:47:57,714 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:57,719 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-12T07:47:57,719 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:57,721 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:57,721 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:57,721 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:57,721 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:57,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:57,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:57,724 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-12T07:47:57,724 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:57,724 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:57,724 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:57,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:57,726 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-12T07:47:57,727 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:57,727 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:57,727 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:57,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:57,737 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/554511800eeb42b390909cb750f3db0e 2024-11-12T07:47:57,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6c3b00a64ca34438a1533ca5ac55b776 is 50, key is test_row_0/C:col10/1731397676711/Put/seqid=0 2024-11-12T07:47:57,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741950_1126 (size=12301) 2024-11-12T07:47:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:47:57,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48080 deadline: 1731397737998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:58,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:58,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:58,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48006 deadline: 1731397738009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48044 deadline: 1731397738009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:58,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48054 deadline: 1731397738009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:47:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48030 deadline: 1731397738010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:47:58,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:47:58,150 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6c3b00a64ca34438a1533ca5ac55b776 2024-11-12T07:47:58,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/3fd7510f623d4a158a9dde3f1b57fd62 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3fd7510f623d4a158a9dde3f1b57fd62 2024-11-12T07:47:58,163 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3fd7510f623d4a158a9dde3f1b57fd62, entries=150, sequenceid=705, filesize=12.0 K 2024-11-12T07:47:58,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/554511800eeb42b390909cb750f3db0e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/554511800eeb42b390909cb750f3db0e 2024-11-12T07:47:58,170 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/554511800eeb42b390909cb750f3db0e, entries=150, sequenceid=705, filesize=12.0 K 2024-11-12T07:47:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/6c3b00a64ca34438a1533ca5ac55b776 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c3b00a64ca34438a1533ca5ac55b776 2024-11-12T07:47:58,176 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c3b00a64ca34438a1533ca5ac55b776, entries=150, sequenceid=705, filesize=12.0 K 2024-11-12T07:47:58,177 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 609f0702d0c4b8c7ee5a4c9652b6127b in 930ms, sequenceid=705, compaction requested=true 2024-11-12T07:47:58,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:58,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:58,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-12T07:47:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-12T07:47:58,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-12T07:47:58,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5460 sec 2024-11-12T07:47:58,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.5510 sec 2024-11-12T07:47:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:58,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T07:47:58,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:58,515 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-12T07:47:58,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:58,515 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:58,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:58,515 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:58,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:58,516 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:58,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:58,520 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-12T07:47:58,520 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:58,520 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,520 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-12T07:47:58,520 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:58,520 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-12T07:47:58,520 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,520 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,520 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:58,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:58,520 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,520 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:58,521 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:58,522 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-12T07:47:58,522 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:47:58,522 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,522 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:58,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/522e3b48b3a845aa9cf4bfd9de16792f is 50, key is test_row_0/A:col10/1731397677365/Put/seqid=0 2024-11-12T07:47:58,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741951_1127 (size=12301) 2024-11-12T07:47:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T07:47:58,780 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:47:58,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T07:47:58,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-12T07:47:58,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-12T07:47:58,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 186 2024-11-12T07:47:58,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-12T07:47:58,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-12T07:47:58,781 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T07:47:58,782 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2710 2024-11-12T07:47:58,782 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2621 2024-11-12T07:47:58,782 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T07:47:58,782 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1168 2024-11-12T07:47:58,782 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3497 rows 2024-11-12T07:47:58,782 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1169 2024-11-12T07:47:58,782 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3505 rows 2024-11-12T07:47:58,783 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:47:58,783 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:47:58,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:47:58,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:47:58,792 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T07:47:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T07:47:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T07:47:58,801 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397678800"}]},"ts":"1731397678800"} 2024-11-12T07:47:58,803 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T07:47:58,803 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T07:47:58,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:47:58,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, UNASSIGN}] 2024-11-12T07:47:58,808 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, UNASSIGN 2024-11-12T07:47:58,810 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=609f0702d0c4b8c7ee5a4c9652b6127b, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:47:58,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, UNASSIGN because future has completed 2024-11-12T07:47:58,812 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:47:58,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:47:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T07:47:58,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/522e3b48b3a845aa9cf4bfd9de16792f 2024-11-12T07:47:58,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/34addb4594644a46b2bfaee3d91afb45 is 50, key is test_row_0/B:col10/1731397677365/Put/seqid=0 2024-11-12T07:47:58,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741952_1128 (size=12301) 2024-11-12T07:47:58,969 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:47:58,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:47:58,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 609f0702d0c4b8c7ee5a4c9652b6127b, disabling compactions & flushes 2024-11-12T07:47:58,969 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T07:47:59,129 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:47:59,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/34addb4594644a46b2bfaee3d91afb45 2024-11-12T07:47:59,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/cf1d3ba8716f47648a1a2323a19dc943 is 50, key is test_row_0/C:col10/1731397677365/Put/seqid=0 2024-11-12T07:47:59,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741953_1129 (size=12301) 2024-11-12T07:47:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T07:47:59,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/cf1d3ba8716f47648a1a2323a19dc943 2024-11-12T07:47:59,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/522e3b48b3a845aa9cf4bfd9de16792f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/522e3b48b3a845aa9cf4bfd9de16792f 2024-11-12T07:47:59,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/522e3b48b3a845aa9cf4bfd9de16792f, entries=150, sequenceid=720, filesize=12.0 K 2024-11-12T07:47:59,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/34addb4594644a46b2bfaee3d91afb45 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/34addb4594644a46b2bfaee3d91afb45 2024-11-12T07:47:59,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/34addb4594644a46b2bfaee3d91afb45, entries=150, sequenceid=720, filesize=12.0 K 2024-11-12T07:47:59,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/cf1d3ba8716f47648a1a2323a19dc943 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/cf1d3ba8716f47648a1a2323a19dc943 2024-11-12T07:47:59,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/cf1d3ba8716f47648a1a2323a19dc943, entries=150, sequenceid=720, filesize=12.0 K 2024-11-12T07:47:59,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=26.84 KB/27480 for 609f0702d0c4b8c7ee5a4c9652b6127b in 1278ms, sequenceid=720, compaction requested=true 2024-11-12T07:47:59,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 609f0702d0c4b8c7ee5a4c9652b6127b: 2024-11-12T07:47:59,792 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:59,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:47:59,792 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:59,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:59,792 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. after waiting 0 ms 2024-11-12T07:47:59,793 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:47:59,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:47:59,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:47:59,793 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. because compaction request was cancelled 2024-11-12T07:47:59,793 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 609f0702d0c4b8c7ee5a4c9652b6127b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T07:47:59,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 609f0702d0c4b8c7ee5a4c9652b6127b:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:47:59,793 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. because compaction request was cancelled 2024-11-12T07:47:59,793 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:A 2024-11-12T07:47:59,793 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:B 2024-11-12T07:47:59,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:47:59,793 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. because compaction request was cancelled 2024-11-12T07:47:59,793 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 609f0702d0c4b8c7ee5a4c9652b6127b:C 2024-11-12T07:47:59,793 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=A 2024-11-12T07:47:59,793 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:59,793 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=B 2024-11-12T07:47:59,793 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:59,793 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 609f0702d0c4b8c7ee5a4c9652b6127b, store=C 2024-11-12T07:47:59,793 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:47:59,797 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/177e3ec7f25f45f8ad1897fee5dbb691 is 50, key is test_row_0/A:col10/1731397678520/Put/seqid=0 2024-11-12T07:47:59,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741954_1130 (size=12301) 2024-11-12T07:47:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T07:48:00,202 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/177e3ec7f25f45f8ad1897fee5dbb691 2024-11-12T07:48:00,216 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/497654b91e6e40aba8493e9c81e1232e is 50, key is test_row_0/B:col10/1731397678520/Put/seqid=0 2024-11-12T07:48:00,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741955_1131 (size=12301) 2024-11-12T07:48:00,622 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/497654b91e6e40aba8493e9c81e1232e 2024-11-12T07:48:00,635 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/2e301ccce2c240489db1a9004c82c5fe is 50, key is test_row_0/C:col10/1731397678520/Put/seqid=0 2024-11-12T07:48:00,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741956_1132 (size=12301) 2024-11-12T07:48:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T07:48:01,039 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/2e301ccce2c240489db1a9004c82c5fe 2024-11-12T07:48:01,045 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/A/177e3ec7f25f45f8ad1897fee5dbb691 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/177e3ec7f25f45f8ad1897fee5dbb691 2024-11-12T07:48:01,049 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/177e3ec7f25f45f8ad1897fee5dbb691, entries=150, sequenceid=727, filesize=12.0 K 2024-11-12T07:48:01,050 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/B/497654b91e6e40aba8493e9c81e1232e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/497654b91e6e40aba8493e9c81e1232e 2024-11-12T07:48:01,054 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/497654b91e6e40aba8493e9c81e1232e, entries=150, sequenceid=727, filesize=12.0 K 2024-11-12T07:48:01,055 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/.tmp/C/2e301ccce2c240489db1a9004c82c5fe as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/2e301ccce2c240489db1a9004c82c5fe 2024-11-12T07:48:01,059 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/2e301ccce2c240489db1a9004c82c5fe, entries=150, sequenceid=727, filesize=12.0 K 2024-11-12T07:48:01,060 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 609f0702d0c4b8c7ee5a4c9652b6127b in 1267ms, sequenceid=727, compaction requested=true 2024-11-12T07:48:01,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/0eb196bd5eb74510a21bfe6866486c22, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f12b0ac12e76435c83af4f09979727fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/4adeb499a63442e88dc9916f219dfd93, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b90d6d80c5c749f98f6ea21cbe5129fa, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/381069df1f264d36b4048efee4234b56, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3adffaa7fa344a8b962b0490ac4ce0e6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/e327853f546a4439a42d08c4ce9c2f30, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/5112868583354f7bb24b234b3370dc1e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/225a12a9d0c64119b9d1fc40b5662059, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/da6e9ff3dc554741a1d20267241e8946, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d0d95fc689154059b285331a0e54d991, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d1b7b25243ab49618a23f325209ce9e4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/6ede51db708244c5946f2c5fc8c07c40, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7484d288c7cf4ea8a418af1118be44a0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f3656d1982e3473eb054b1b48efeff24, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/efbbc729a84a40a2bdb7270cfe89ead8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b21baa4e9e3249a3bbb0014143bcec41, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/c9934960f6a5481f8cdb9d6a6d9ebbef, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/ef7371e4684f475592cfb8f74bec19db, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/952986a175134bee8deb2fab9aba3d48, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/455c33e9527c42f68cd33782dd4c848f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/89963c140f8044cc8ca76d2301dae6bc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8888487f79894487a68406d5194ac70d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8d8b3c40e342441cb2617ea7be40b534, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/83eea909a2bd4767a4c451d88ddf3c8e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/63cbec73df6b4f4d80936ae17218e04c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7ab1957a31e3416487134af6069467bb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8c7a5f10b62148a6991e3a8fd127e0c2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/fbf34fd4cfa247719a898b74dc7e3877, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d828b6a7fd3c45b9a84b869d6afdb327, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d78b671c38ec4267b011e147764e1585, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9a23524f3d2b428da946c24069319f96, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/217891ea93144abb9189f05bfdc64445, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/834f18957a15460f877fe343059e1ded, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9e929d49391042f9aaa10622ab108200] to archive 2024-11-12T07:48:01,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:48:01,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/0eb196bd5eb74510a21bfe6866486c22 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/0eb196bd5eb74510a21bfe6866486c22 2024-11-12T07:48:01,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f12b0ac12e76435c83af4f09979727fb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f12b0ac12e76435c83af4f09979727fb 2024-11-12T07:48:01,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/4adeb499a63442e88dc9916f219dfd93 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/4adeb499a63442e88dc9916f219dfd93 2024-11-12T07:48:01,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b90d6d80c5c749f98f6ea21cbe5129fa to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b90d6d80c5c749f98f6ea21cbe5129fa 2024-11-12T07:48:01,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/381069df1f264d36b4048efee4234b56 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/381069df1f264d36b4048efee4234b56 2024-11-12T07:48:01,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3adffaa7fa344a8b962b0490ac4ce0e6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3adffaa7fa344a8b962b0490ac4ce0e6 2024-11-12T07:48:01,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/e327853f546a4439a42d08c4ce9c2f30 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/e327853f546a4439a42d08c4ce9c2f30 2024-11-12T07:48:01,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/5112868583354f7bb24b234b3370dc1e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/5112868583354f7bb24b234b3370dc1e 2024-11-12T07:48:01,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/225a12a9d0c64119b9d1fc40b5662059 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/225a12a9d0c64119b9d1fc40b5662059 2024-11-12T07:48:01,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/da6e9ff3dc554741a1d20267241e8946 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/da6e9ff3dc554741a1d20267241e8946 2024-11-12T07:48:01,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d0d95fc689154059b285331a0e54d991 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d0d95fc689154059b285331a0e54d991 2024-11-12T07:48:01,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d1b7b25243ab49618a23f325209ce9e4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d1b7b25243ab49618a23f325209ce9e4 2024-11-12T07:48:01,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/6ede51db708244c5946f2c5fc8c07c40 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/6ede51db708244c5946f2c5fc8c07c40 2024-11-12T07:48:01,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7484d288c7cf4ea8a418af1118be44a0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7484d288c7cf4ea8a418af1118be44a0 2024-11-12T07:48:01,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f3656d1982e3473eb054b1b48efeff24 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/f3656d1982e3473eb054b1b48efeff24 2024-11-12T07:48:01,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/efbbc729a84a40a2bdb7270cfe89ead8 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/efbbc729a84a40a2bdb7270cfe89ead8 2024-11-12T07:48:01,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b21baa4e9e3249a3bbb0014143bcec41 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/b21baa4e9e3249a3bbb0014143bcec41 2024-11-12T07:48:01,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/c9934960f6a5481f8cdb9d6a6d9ebbef to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/c9934960f6a5481f8cdb9d6a6d9ebbef 2024-11-12T07:48:01,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/ef7371e4684f475592cfb8f74bec19db to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/ef7371e4684f475592cfb8f74bec19db 2024-11-12T07:48:01,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/952986a175134bee8deb2fab9aba3d48 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/952986a175134bee8deb2fab9aba3d48 2024-11-12T07:48:01,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/455c33e9527c42f68cd33782dd4c848f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/455c33e9527c42f68cd33782dd4c848f 2024-11-12T07:48:01,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/89963c140f8044cc8ca76d2301dae6bc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/89963c140f8044cc8ca76d2301dae6bc 2024-11-12T07:48:01,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8888487f79894487a68406d5194ac70d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8888487f79894487a68406d5194ac70d 2024-11-12T07:48:01,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8d8b3c40e342441cb2617ea7be40b534 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8d8b3c40e342441cb2617ea7be40b534 2024-11-12T07:48:01,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/83eea909a2bd4767a4c451d88ddf3c8e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/83eea909a2bd4767a4c451d88ddf3c8e 2024-11-12T07:48:01,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/63cbec73df6b4f4d80936ae17218e04c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/63cbec73df6b4f4d80936ae17218e04c 2024-11-12T07:48:01,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7ab1957a31e3416487134af6069467bb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/7ab1957a31e3416487134af6069467bb 2024-11-12T07:48:01,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8c7a5f10b62148a6991e3a8fd127e0c2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/8c7a5f10b62148a6991e3a8fd127e0c2 2024-11-12T07:48:01,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/fbf34fd4cfa247719a898b74dc7e3877 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/fbf34fd4cfa247719a898b74dc7e3877 2024-11-12T07:48:01,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d828b6a7fd3c45b9a84b869d6afdb327 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d828b6a7fd3c45b9a84b869d6afdb327 2024-11-12T07:48:01,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d78b671c38ec4267b011e147764e1585 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/d78b671c38ec4267b011e147764e1585 2024-11-12T07:48:01,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9a23524f3d2b428da946c24069319f96 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9a23524f3d2b428da946c24069319f96 2024-11-12T07:48:01,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/217891ea93144abb9189f05bfdc64445 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/217891ea93144abb9189f05bfdc64445 2024-11-12T07:48:01,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/834f18957a15460f877fe343059e1ded to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/834f18957a15460f877fe343059e1ded 2024-11-12T07:48:01,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9e929d49391042f9aaa10622ab108200 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/9e929d49391042f9aaa10622ab108200 2024-11-12T07:48:01,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ead7a74461b4f6a8c59b119c3b3bb4a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/9268c70da2f94ad5a917547b5f466c04, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/57579a172b1347d8be381413928eafde, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/61e60a18fa5c4e96b5624c5bff0c4581, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/620f91b9c01a4c33b4007efa3dfee026, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e9dbdd7586a64dc99d4d3d07344684f1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fe8990c026994ab093e6b039e71fbbff, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e28994dd76f441a8870de1893d2e6bc2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f0494dfc5ea64f0b9f10e0b1da98146d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cb0d342d76fd44f599163141d8173cd0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/c37f4b1ffcf540039c5f9ecaba8d3a8b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fa050feebdb24cc6b4dea59de14d8b46, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e37f79ffc2e1441f86ece3ee6d52109e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ce45c7dc47a4ac2b8351fd5c73d8003, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/093e228f99cb4cdeb3a8c4000cf0cdb6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d799b5e289154e68a39acce509b82a90, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/81f3000f80b040d18187bebbf383755c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/63df74214c914943bc8b637fe12d32d1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/026f26240b0a411c9d07eaabeded07cf, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f14a90ae9e0e4957b82e43f6f7824256, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/16533644389c4143aae666e5ea1910cb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6d1ab49d192a4db8bdf85477c5886adb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/de84b6ec117c40c080a5dd955b40c860, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/1ec0261a1bfb4ad5b059ff22c2c82c86, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/8d742d80fb2b4a42ae692cde6cc11d7c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d7445b491dc74061be13e4310dccc2a9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/4b0e5affbcf34d6ea44bf42b235853e9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cfe637dc17d3492a84631483e01cc5cd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/80dbe372f14349879c10d24d76192d40, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/207680040e764953886476e3ad6593fd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/41fe9b9bac65439e858d10a1d6299d92, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fcc0ae5da9bc4885b6e760be7c9fe0dd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/01501e24f65a42928112cfa0efab9c84, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/78bf0a03c33f4877956862c827f55e01] to archive 2024-11-12T07:48:01,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:48:01,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ead7a74461b4f6a8c59b119c3b3bb4a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ead7a74461b4f6a8c59b119c3b3bb4a 2024-11-12T07:48:01,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/9268c70da2f94ad5a917547b5f466c04 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/9268c70da2f94ad5a917547b5f466c04 2024-11-12T07:48:01,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/57579a172b1347d8be381413928eafde to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/57579a172b1347d8be381413928eafde 2024-11-12T07:48:01,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/61e60a18fa5c4e96b5624c5bff0c4581 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/61e60a18fa5c4e96b5624c5bff0c4581 2024-11-12T07:48:01,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/620f91b9c01a4c33b4007efa3dfee026 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/620f91b9c01a4c33b4007efa3dfee026 2024-11-12T07:48:01,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e9dbdd7586a64dc99d4d3d07344684f1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e9dbdd7586a64dc99d4d3d07344684f1 2024-11-12T07:48:01,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fe8990c026994ab093e6b039e71fbbff to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fe8990c026994ab093e6b039e71fbbff 2024-11-12T07:48:01,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e28994dd76f441a8870de1893d2e6bc2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e28994dd76f441a8870de1893d2e6bc2 2024-11-12T07:48:01,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f0494dfc5ea64f0b9f10e0b1da98146d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f0494dfc5ea64f0b9f10e0b1da98146d 2024-11-12T07:48:01,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cb0d342d76fd44f599163141d8173cd0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cb0d342d76fd44f599163141d8173cd0 2024-11-12T07:48:01,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/c37f4b1ffcf540039c5f9ecaba8d3a8b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/c37f4b1ffcf540039c5f9ecaba8d3a8b 2024-11-12T07:48:01,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fa050feebdb24cc6b4dea59de14d8b46 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fa050feebdb24cc6b4dea59de14d8b46 2024-11-12T07:48:01,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e37f79ffc2e1441f86ece3ee6d52109e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/e37f79ffc2e1441f86ece3ee6d52109e 2024-11-12T07:48:01,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ce45c7dc47a4ac2b8351fd5c73d8003 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/5ce45c7dc47a4ac2b8351fd5c73d8003 2024-11-12T07:48:01,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/093e228f99cb4cdeb3a8c4000cf0cdb6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/093e228f99cb4cdeb3a8c4000cf0cdb6 2024-11-12T07:48:01,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d799b5e289154e68a39acce509b82a90 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d799b5e289154e68a39acce509b82a90 2024-11-12T07:48:01,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/81f3000f80b040d18187bebbf383755c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/81f3000f80b040d18187bebbf383755c 2024-11-12T07:48:01,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/63df74214c914943bc8b637fe12d32d1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/63df74214c914943bc8b637fe12d32d1 2024-11-12T07:48:01,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/026f26240b0a411c9d07eaabeded07cf to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/026f26240b0a411c9d07eaabeded07cf 2024-11-12T07:48:01,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f14a90ae9e0e4957b82e43f6f7824256 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/f14a90ae9e0e4957b82e43f6f7824256 2024-11-12T07:48:01,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6f1f9a11ab6e4b9dbce1ec863f8e78b6 2024-11-12T07:48:01,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/16533644389c4143aae666e5ea1910cb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/16533644389c4143aae666e5ea1910cb 2024-11-12T07:48:01,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6d1ab49d192a4db8bdf85477c5886adb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/6d1ab49d192a4db8bdf85477c5886adb 2024-11-12T07:48:01,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/de84b6ec117c40c080a5dd955b40c860 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/de84b6ec117c40c080a5dd955b40c860 2024-11-12T07:48:01,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/1ec0261a1bfb4ad5b059ff22c2c82c86 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/1ec0261a1bfb4ad5b059ff22c2c82c86 2024-11-12T07:48:01,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/8d742d80fb2b4a42ae692cde6cc11d7c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/8d742d80fb2b4a42ae692cde6cc11d7c 2024-11-12T07:48:01,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d7445b491dc74061be13e4310dccc2a9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/d7445b491dc74061be13e4310dccc2a9 2024-11-12T07:48:01,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/4b0e5affbcf34d6ea44bf42b235853e9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/4b0e5affbcf34d6ea44bf42b235853e9 2024-11-12T07:48:01,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cfe637dc17d3492a84631483e01cc5cd to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/cfe637dc17d3492a84631483e01cc5cd 2024-11-12T07:48:01,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/80dbe372f14349879c10d24d76192d40 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/80dbe372f14349879c10d24d76192d40 2024-11-12T07:48:01,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/207680040e764953886476e3ad6593fd to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/207680040e764953886476e3ad6593fd 2024-11-12T07:48:01,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/41fe9b9bac65439e858d10a1d6299d92 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/41fe9b9bac65439e858d10a1d6299d92 2024-11-12T07:48:01,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fcc0ae5da9bc4885b6e760be7c9fe0dd to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/fcc0ae5da9bc4885b6e760be7c9fe0dd 2024-11-12T07:48:01,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/01501e24f65a42928112cfa0efab9c84 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/01501e24f65a42928112cfa0efab9c84 2024-11-12T07:48:01,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/78bf0a03c33f4877956862c827f55e01 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/78bf0a03c33f4877956862c827f55e01 2024-11-12T07:48:01,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/23f9f1359e3b460bb8cd5bd071d36e3e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/983cfdadc7f94ce399979f47167e86d0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a00bbb899fa04979a977965aa976a453, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39013d3400e94273999bac14cc082984, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5f15689b442b47c78ba9e2e1e95cd974, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/dbfea798e89a44c193894552ed7a6b18, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d579b4eece7742e483504642cd035903, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/f06bd492993e487f8d91397cffdd768f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/26d3630e48f44651aded42c41de58e1f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/882ee1b78bb648adbeb80d743d4b8f1b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9c8a39518ce942e3ad00ae1f94602664, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c7a04fe99ec441cba187aaca061daae, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f69df03b14747a39b8974a469e6da41, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/16ed225157194606a448c3df62cee5be, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/530639371ed4456798a343e64cefa327, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/b83a4d73647847d8b877ae0c415e4ecb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/fc8c1dd551d04825863118b1b78e4d23, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e934615fe9504f3182b42da022a397f0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/8d97dd6b2281460bba6abd55a38f308f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d2b0421c5ec74b3fb5c9e5fce5e1bdce, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/80addfb44e1b44d9a1f83d5e11909a05, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/99286b41893148429bbd5f0bccbf1367, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9fba11752c31481abf6ef67736157f02, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39fad9f8fed24dc3ade49a561cf1c89c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/408dc1b8ebad4896be20b5bd4edf4ddc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca6aadfed2c5464b8cbd0e83f31d5437, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca815d49f19e44f49a2a46743344591e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e22619d839fa4024afbed99313a96c95, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/448a5bc02447476a94595a8ec9130194, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a2d60b50be8c4a66bc3aa7e931518d17, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f563ee1982d4b4db0905866428e9d8e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/1c105333abeb42d3967c50d8e95e02ef, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d28f5cb4dee543d4a917f947cbcf07d7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6da4a74679464637885d73dc45fb647e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5868c203b7174de2a8b1b0edaa11aca6] to archive 2024-11-12T07:48:01,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:48:01,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/23f9f1359e3b460bb8cd5bd071d36e3e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/23f9f1359e3b460bb8cd5bd071d36e3e 2024-11-12T07:48:01,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/983cfdadc7f94ce399979f47167e86d0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/983cfdadc7f94ce399979f47167e86d0 2024-11-12T07:48:01,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a00bbb899fa04979a977965aa976a453 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a00bbb899fa04979a977965aa976a453 2024-11-12T07:48:01,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39013d3400e94273999bac14cc082984 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39013d3400e94273999bac14cc082984 2024-11-12T07:48:01,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5f15689b442b47c78ba9e2e1e95cd974 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5f15689b442b47c78ba9e2e1e95cd974 2024-11-12T07:48:01,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/dbfea798e89a44c193894552ed7a6b18 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/dbfea798e89a44c193894552ed7a6b18 2024-11-12T07:48:01,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d579b4eece7742e483504642cd035903 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d579b4eece7742e483504642cd035903 2024-11-12T07:48:01,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/f06bd492993e487f8d91397cffdd768f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/f06bd492993e487f8d91397cffdd768f 2024-11-12T07:48:01,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/26d3630e48f44651aded42c41de58e1f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/26d3630e48f44651aded42c41de58e1f 2024-11-12T07:48:01,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/882ee1b78bb648adbeb80d743d4b8f1b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/882ee1b78bb648adbeb80d743d4b8f1b 2024-11-12T07:48:01,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9c8a39518ce942e3ad00ae1f94602664 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9c8a39518ce942e3ad00ae1f94602664 2024-11-12T07:48:01,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c7a04fe99ec441cba187aaca061daae to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c7a04fe99ec441cba187aaca061daae 2024-11-12T07:48:01,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f69df03b14747a39b8974a469e6da41 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f69df03b14747a39b8974a469e6da41 2024-11-12T07:48:01,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/16ed225157194606a448c3df62cee5be to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/16ed225157194606a448c3df62cee5be 2024-11-12T07:48:01,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/530639371ed4456798a343e64cefa327 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/530639371ed4456798a343e64cefa327 2024-11-12T07:48:01,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/b83a4d73647847d8b877ae0c415e4ecb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/b83a4d73647847d8b877ae0c415e4ecb 2024-11-12T07:48:01,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/fc8c1dd551d04825863118b1b78e4d23 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/fc8c1dd551d04825863118b1b78e4d23 2024-11-12T07:48:01,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e934615fe9504f3182b42da022a397f0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e934615fe9504f3182b42da022a397f0 2024-11-12T07:48:01,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/8d97dd6b2281460bba6abd55a38f308f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/8d97dd6b2281460bba6abd55a38f308f 2024-11-12T07:48:01,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d2b0421c5ec74b3fb5c9e5fce5e1bdce to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d2b0421c5ec74b3fb5c9e5fce5e1bdce 2024-11-12T07:48:01,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/80addfb44e1b44d9a1f83d5e11909a05 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/80addfb44e1b44d9a1f83d5e11909a05 2024-11-12T07:48:01,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/99286b41893148429bbd5f0bccbf1367 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/99286b41893148429bbd5f0bccbf1367 2024-11-12T07:48:01,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9fba11752c31481abf6ef67736157f02 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/9fba11752c31481abf6ef67736157f02 2024-11-12T07:48:01,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39fad9f8fed24dc3ade49a561cf1c89c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/39fad9f8fed24dc3ade49a561cf1c89c 2024-11-12T07:48:01,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/408dc1b8ebad4896be20b5bd4edf4ddc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/408dc1b8ebad4896be20b5bd4edf4ddc 2024-11-12T07:48:01,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca6aadfed2c5464b8cbd0e83f31d5437 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca6aadfed2c5464b8cbd0e83f31d5437 2024-11-12T07:48:01,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca815d49f19e44f49a2a46743344591e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/ca815d49f19e44f49a2a46743344591e 2024-11-12T07:48:01,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e22619d839fa4024afbed99313a96c95 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/e22619d839fa4024afbed99313a96c95 2024-11-12T07:48:01,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/448a5bc02447476a94595a8ec9130194 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/448a5bc02447476a94595a8ec9130194 2024-11-12T07:48:01,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a2d60b50be8c4a66bc3aa7e931518d17 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/a2d60b50be8c4a66bc3aa7e931518d17 2024-11-12T07:48:01,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f563ee1982d4b4db0905866428e9d8e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6f563ee1982d4b4db0905866428e9d8e 2024-11-12T07:48:01,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/1c105333abeb42d3967c50d8e95e02ef to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/1c105333abeb42d3967c50d8e95e02ef 2024-11-12T07:48:01,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d28f5cb4dee543d4a917f947cbcf07d7 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/d28f5cb4dee543d4a917f947cbcf07d7 2024-11-12T07:48:01,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6da4a74679464637885d73dc45fb647e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6da4a74679464637885d73dc45fb647e 2024-11-12T07:48:01,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5868c203b7174de2a8b1b0edaa11aca6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/5868c203b7174de2a8b1b0edaa11aca6 2024-11-12T07:48:01,230 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/recovered.edits/730.seqid, newMaxSeqId=730, maxSeqId=1 2024-11-12T07:48:01,230 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. 2024-11-12T07:48:01,231 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 609f0702d0c4b8c7ee5a4c9652b6127b: Waiting for close lock at 1731397678969Running coprocessor pre-close hooks at 1731397678969Disabling compacts and flushes for region at 1731397678969Disabling writes for close at 1731397679792 (+823 ms)Obtaining lock to block concurrent updates at 1731397679793 (+1 ms)Preparing flush snapshotting stores in 609f0702d0c4b8c7ee5a4c9652b6127b at 1731397679793Finished memstore snapshotting TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731397679793Flushing stores of TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b. at 1731397679794 (+1 ms)Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/A: creating writer at 1731397679794Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/A: appending metadata at 1731397679797 (+3 ms)Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/A: closing flushed file at 1731397679797Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/B: creating writer at 1731397680212 (+415 ms)Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/B: appending metadata at 1731397680215 (+3 ms)Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/B: closing flushed file at 1731397680215Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/C: creating writer at 1731397680632 (+417 ms)Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/C: appending metadata at 1731397680634 (+2 ms)Flushing 609f0702d0c4b8c7ee5a4c9652b6127b/C: closing flushed file at 1731397680634Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f3a376e: reopening flushed file at 1731397681044 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@730c9110: reopening flushed file at 1731397681050 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ac90113: reopening flushed file at 1731397681055 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 609f0702d0c4b8c7ee5a4c9652b6127b in 1267ms, sequenceid=727, compaction requested=true at 1731397681060 (+5 ms)Writing region close event to WAL at 1731397681226 (+166 ms)Running coprocessor post-close hooks at 1731397681230 (+4 ms)Closed at 1731397681230 2024-11-12T07:48:01,233 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:48:01,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=609f0702d0c4b8c7ee5a4c9652b6127b, regionState=CLOSED 2024-11-12T07:48:01,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:48:01,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T07:48:01,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 609f0702d0c4b8c7ee5a4c9652b6127b, server=f9fc96254ef8,44299,1731397650797 in 2.4240 sec 2024-11-12T07:48:01,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-12T07:48:01,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=609f0702d0c4b8c7ee5a4c9652b6127b, UNASSIGN in 2.4300 sec 2024-11-12T07:48:01,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-12T07:48:01,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4370 sec 2024-11-12T07:48:01,243 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397681243"}]},"ts":"1731397681243"} 2024-11-12T07:48:01,245 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T07:48:01,245 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T07:48:01,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4500 sec 2024-11-12T07:48:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-12T07:48:02,970 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:02,979 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T07:48:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:02,985 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-12T07:48:02,986 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:02,992 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:48:02,996 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/recovered.edits] 2024-11-12T07:48:02,999 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/177e3ec7f25f45f8ad1897fee5dbb691 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/177e3ec7f25f45f8ad1897fee5dbb691 2024-11-12T07:48:03,001 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/21c3ddade7e843df96ab5a27ff23ab0c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/21c3ddade7e843df96ab5a27ff23ab0c 2024-11-12T07:48:03,002 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3fd7510f623d4a158a9dde3f1b57fd62 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/3fd7510f623d4a158a9dde3f1b57fd62 2024-11-12T07:48:03,003 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/522e3b48b3a845aa9cf4bfd9de16792f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/522e3b48b3a845aa9cf4bfd9de16792f 2024-11-12T07:48:03,005 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/829612420aca40e4b27185fee27f4585 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/A/829612420aca40e4b27185fee27f4585 2024-11-12T07:48:03,007 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/34addb4594644a46b2bfaee3d91afb45 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/34addb4594644a46b2bfaee3d91afb45 2024-11-12T07:48:03,008 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/497654b91e6e40aba8493e9c81e1232e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/497654b91e6e40aba8493e9c81e1232e 2024-11-12T07:48:03,009 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/4ffaeb5daf0643f7917531b082825f30 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/4ffaeb5daf0643f7917531b082825f30 2024-11-12T07:48:03,010 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/554511800eeb42b390909cb750f3db0e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/554511800eeb42b390909cb750f3db0e 2024-11-12T07:48:03,012 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/b885eb8b2d2e4173a1cfd17ccda92da8 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/B/b885eb8b2d2e4173a1cfd17ccda92da8 2024-11-12T07:48:03,014 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/0de975c60d9a4a81acb63fc2cf23ebcb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/0de975c60d9a4a81acb63fc2cf23ebcb 2024-11-12T07:48:03,015 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/0f33da7b9e344e14a4baff86ed5955af to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/0f33da7b9e344e14a4baff86ed5955af 2024-11-12T07:48:03,016 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/2e301ccce2c240489db1a9004c82c5fe to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/2e301ccce2c240489db1a9004c82c5fe 2024-11-12T07:48:03,017 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c3b00a64ca34438a1533ca5ac55b776 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/6c3b00a64ca34438a1533ca5ac55b776 2024-11-12T07:48:03,018 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/cf1d3ba8716f47648a1a2323a19dc943 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/C/cf1d3ba8716f47648a1a2323a19dc943 2024-11-12T07:48:03,021 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/recovered.edits/730.seqid to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b/recovered.edits/730.seqid 2024-11-12T07:48:03,022 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/609f0702d0c4b8c7ee5a4c9652b6127b 2024-11-12T07:48:03,022 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T07:48:03,027 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:03,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-12T07:48:03,036 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T07:48:03,065 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T07:48:03,066 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:03,067 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T07:48:03,067 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731397683067"}]},"ts":"9223372036854775807"} 2024-11-12T07:48:03,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T07:48:03,071 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 609f0702d0c4b8c7ee5a4c9652b6127b, NAME => 'TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T07:48:03,071 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T07:48:03,071 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731397683071"}]},"ts":"9223372036854775807"} 2024-11-12T07:48:03,074 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T07:48:03,075 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:03,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-12T07:48:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-12T07:48:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T07:48:03,099 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:03,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;f9fc96254ef8:44299-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1287358721_22 at /127.0.0.1:51508 [Waiting for operation #379] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_462226526_22 at /127.0.0.1:41908 [Waiting for operation #363] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 238) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6055 (was 6622) 2024-11-12T07:48:03,120 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=6054 2024-11-12T07:48:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:48:03,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:48:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:03,124 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:48:03,125 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:03,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-12T07:48:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T07:48:03,126 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:48:03,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741957_1133 (size=1061) 2024-11-12T07:48:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T07:48:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T07:48:03,535 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f 2024-11-12T07:48:03,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741958_1134 (size=53) 2024-11-12T07:48:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T07:48:03,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:48:03,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 299b057d313257fac3b10771c8ca0f20, disabling compactions & flushes 2024-11-12T07:48:03,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:03,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:03,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. after waiting 0 ms 2024-11-12T07:48:03,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:03,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:03,946 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 299b057d313257fac3b10771c8ca0f20: Waiting for close lock at 1731397683946Disabling compacts and flushes for region at 1731397683946Disabling writes for close at 1731397683946Writing region close event to WAL at 1731397683946Closed at 1731397683946 2024-11-12T07:48:03,949 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:48:03,950 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731397683949"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397683949"}]},"ts":"1731397683949"} 2024-11-12T07:48:03,952 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:48:03,953 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:48:03,953 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397683953"}]},"ts":"1731397683953"} 2024-11-12T07:48:03,955 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T07:48:03,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, ASSIGN}] 2024-11-12T07:48:03,956 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, ASSIGN 2024-11-12T07:48:03,957 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, ASSIGN; state=OFFLINE, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=false 2024-11-12T07:48:04,108 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=299b057d313257fac3b10771c8ca0f20, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:04,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, ASSIGN because future has completed 2024-11-12T07:48:04,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:48:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T07:48:04,270 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:04,270 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:48:04,270 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,270 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:48:04,271 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,271 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,273 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,274 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:04,275 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299b057d313257fac3b10771c8ca0f20 columnFamilyName A 2024-11-12T07:48:04,275 DEBUG [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:04,275 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(327): Store=299b057d313257fac3b10771c8ca0f20/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:04,276 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,277 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:04,277 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299b057d313257fac3b10771c8ca0f20 columnFamilyName B 2024-11-12T07:48:04,277 DEBUG [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:04,277 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(327): Store=299b057d313257fac3b10771c8ca0f20/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:04,278 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,279 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:04,279 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299b057d313257fac3b10771c8ca0f20 columnFamilyName C 2024-11-12T07:48:04,279 DEBUG [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:04,279 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(327): Store=299b057d313257fac3b10771c8ca0f20/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:04,280 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:04,280 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,280 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,281 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,282 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,282 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,282 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:48:04,284 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,286 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:48:04,286 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 299b057d313257fac3b10771c8ca0f20; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70445992, jitterRate=0.04972708225250244}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:48:04,286 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:04,287 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 299b057d313257fac3b10771c8ca0f20: Running coprocessor pre-open hook at 1731397684271Writing region info on filesystem at 1731397684271Initializing all the Stores at 1731397684272 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397684272Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397684272Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397684272Cleaning up temporary data from old regions at 1731397684282 (+10 ms)Running coprocessor post-open hooks at 1731397684286 (+4 ms)Region opened successfully at 1731397684287 (+1 ms) 2024-11-12T07:48:04,288 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., pid=34, masterSystemTime=1731397684263 2024-11-12T07:48:04,290 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:04,290 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:04,291 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=299b057d313257fac3b10771c8ca0f20, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:04,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:48:04,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-12T07:48:04,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 in 183 msec 2024-11-12T07:48:04,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-12T07:48:04,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, ASSIGN in 341 msec 2024-11-12T07:48:04,299 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:48:04,299 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397684299"}]},"ts":"1731397684299"} 2024-11-12T07:48:04,300 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T07:48:04,301 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:48:04,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-12T07:48:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-12T07:48:05,280 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:05,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:05,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:05,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:05,285 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60847fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:05,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:05,289 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccf8029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:05,291 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:05,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:05,294 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:48:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,43319,1731397650023 2024-11-12T07:48:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@227cf768 2024-11-12T07:48:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:48:05,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:48:05,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:48:05,301 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:48:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:05,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741959_1135 (size=1097) 2024-11-12T07:48:05,721 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T07:48:05,721 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T07:48:05,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:48:05,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, REOPEN/MOVE}] 2024-11-12T07:48:05,735 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, REOPEN/MOVE 2024-11-12T07:48:05,736 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=299b057d313257fac3b10771c8ca0f20, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:05,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, REOPEN/MOVE because future has completed 2024-11-12T07:48:05,738 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:48:05,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:48:05,891 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:05,892 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:48:05,892 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 299b057d313257fac3b10771c8ca0f20, disabling compactions & flushes 2024-11-12T07:48:05,892 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:05,893 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:05,893 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. after waiting 0 ms 2024-11-12T07:48:05,893 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:05,902 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T07:48:05,902 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:05,902 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 299b057d313257fac3b10771c8ca0f20: Waiting for close lock at 1731397685892Running coprocessor pre-close hooks at 1731397685892Disabling compacts and flushes for region at 1731397685892Disabling writes for close at 1731397685893 (+1 ms)Writing region close event to WAL at 1731397685894 (+1 ms)Running coprocessor post-close hooks at 1731397685902 (+8 ms)Closed at 1731397685902 2024-11-12T07:48:05,903 WARN [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 299b057d313257fac3b10771c8ca0f20 to self. 2024-11-12T07:48:05,905 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:05,906 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=299b057d313257fac3b10771c8ca0f20, regionState=CLOSED 2024-11-12T07:48:05,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:48:05,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-12T07:48:05,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 in 171 msec 2024-11-12T07:48:05,911 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, REOPEN/MOVE; state=CLOSED, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=true 2024-11-12T07:48:06,063 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=299b057d313257fac3b10771c8ca0f20, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, REOPEN/MOVE because future has completed 2024-11-12T07:48:06,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:48:06,225 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,225 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:48:06,225 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,225 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:48:06,225 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,226 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,229 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,230 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:06,235 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299b057d313257fac3b10771c8ca0f20 columnFamilyName A 2024-11-12T07:48:06,237 DEBUG [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:06,238 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(327): Store=299b057d313257fac3b10771c8ca0f20/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:06,239 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,239 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:06,240 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299b057d313257fac3b10771c8ca0f20 columnFamilyName B 2024-11-12T07:48:06,240 DEBUG [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:06,240 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(327): Store=299b057d313257fac3b10771c8ca0f20/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:06,240 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,241 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:06,241 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 299b057d313257fac3b10771c8ca0f20 columnFamilyName C 2024-11-12T07:48:06,241 DEBUG [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:06,242 INFO [StoreOpener-299b057d313257fac3b10771c8ca0f20-1 {}] regionserver.HStore(327): Store=299b057d313257fac3b10771c8ca0f20/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:06,242 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,242 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,243 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,244 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,245 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,245 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,245 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:48:06,246 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,247 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 299b057d313257fac3b10771c8ca0f20; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69484527, jitterRate=0.035400137305259705}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:48:06,247 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,248 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 299b057d313257fac3b10771c8ca0f20: Running coprocessor pre-open hook at 1731397686226Writing region info on filesystem at 1731397686226Initializing all the Stores at 1731397686227 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397686227Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397686227Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397686227Cleaning up temporary data from old regions at 1731397686245 (+18 ms)Running coprocessor post-open hooks at 1731397686247 (+2 ms)Region opened successfully at 1731397686248 (+1 ms) 2024-11-12T07:48:06,249 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., pid=39, masterSystemTime=1731397686221 2024-11-12T07:48:06,251 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,251 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,251 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=299b057d313257fac3b10771c8ca0f20, regionState=OPEN, openSeqNum=5, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:48:06,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-12T07:48:06,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 in 186 msec 2024-11-12T07:48:06,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-12T07:48:06,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, REOPEN/MOVE in 521 msec 2024-11-12T07:48:06,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-12T07:48:06,260 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 533 msec 2024-11-12T07:48:06,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 958 msec 2024-11-12T07:48:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-12T07:48:06,267 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:06,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,269 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bec7c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,270 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,271 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a33850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,274 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405ca7f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204a5fb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,278 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f76cf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,280 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426626b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,282 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc8b1b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,284 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c449ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,286 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e2d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,288 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6ef827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,290 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47571109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,292 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab690fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,294 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41642a85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,295 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771a0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,297 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c7e2da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,298 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22644b0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:06,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:06,300 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8a4428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:06,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dfab01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:06,303 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:06,304 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,304 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-12T07:48:06,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,305 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:06,306 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T07:48:06,307 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,307 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,307 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,308 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,309 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,310 DEBUG [Thread-686 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,310 DEBUG [Thread-686 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,311 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:06,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,314 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:06,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:48:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:48:06,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:06,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:06,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:06,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:06,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:06,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:06,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126d32e1f24f49488f8767b4727d248e0e_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397686336/Put/seqid=0 2024-11-12T07:48:06,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397746376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741960_1136 (size=19474) 2024-11-12T07:48:06,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397746385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397746388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397746391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397746391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T07:48:06,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T07:48:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397746498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397746509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397746510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397746510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397746518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T07:48:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:06,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T07:48:06,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397746709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397746718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397746719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397746719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:06,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397746738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:06,768 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:48:06,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T07:48:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,800 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:06,806 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126d32e1f24f49488f8767b4727d248e0e_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d32e1f24f49488f8767b4727d248e0e_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:06,807 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d06569a02fd949609e62d5e23c6309de, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:06,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d06569a02fd949609e62d5e23c6309de is 175, key is test_row_0/A:col10/1731397686336/Put/seqid=0 2024-11-12T07:48:06,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741961_1137 (size=56733) 2024-11-12T07:48:06,829 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d06569a02fd949609e62d5e23c6309de 2024-11-12T07:48:06,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4531087f146e473691874f4ec27314fd is 50, key is test_row_0/B:col10/1731397686336/Put/seqid=0 2024-11-12T07:48:06,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741962_1138 (size=12001) 2024-11-12T07:48:06,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T07:48:06,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:06,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:06,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T07:48:07,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397747018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397747037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397747038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397747038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397747048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T07:48:07,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:07,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:07,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:07,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:07,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:07,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T07:48:07,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:07,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:07,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:07,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:07,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:07,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4531087f146e473691874f4ec27314fd 2024-11-12T07:48:07,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d5d456b5595a44d89e04b3c94801190c is 50, key is test_row_0/C:col10/1731397686336/Put/seqid=0 2024-11-12T07:48:07,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741963_1139 (size=12001) 2024-11-12T07:48:07,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d5d456b5595a44d89e04b3c94801190c 2024-11-12T07:48:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d06569a02fd949609e62d5e23c6309de as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d06569a02fd949609e62d5e23c6309de 2024-11-12T07:48:07,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d06569a02fd949609e62d5e23c6309de, entries=300, sequenceid=17, filesize=55.4 K 2024-11-12T07:48:07,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4531087f146e473691874f4ec27314fd as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4531087f146e473691874f4ec27314fd 2024-11-12T07:48:07,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4531087f146e473691874f4ec27314fd, entries=150, sequenceid=17, filesize=11.7 K 2024-11-12T07:48:07,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d5d456b5595a44d89e04b3c94801190c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d5d456b5595a44d89e04b3c94801190c 2024-11-12T07:48:07,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d5d456b5595a44d89e04b3c94801190c, entries=150, sequenceid=17, filesize=11.7 K 2024-11-12T07:48:07,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 299b057d313257fac3b10771c8ca0f20 in 1032ms, sequenceid=17, compaction requested=false 2024-11-12T07:48:07,370 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T07:48:07,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:07,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T07:48:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:07,392 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T07:48:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ca5eebf693be4498b43e1eab36130695_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397686388/Put/seqid=0 2024-11-12T07:48:07,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741964_1140 (size=12154) 2024-11-12T07:48:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:07,432 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ca5eebf693be4498b43e1eab36130695_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ca5eebf693be4498b43e1eab36130695_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:07,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/7215861668b74c09a525513166a85c08, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/7215861668b74c09a525513166a85c08 is 175, key is test_row_0/A:col10/1731397686388/Put/seqid=0 2024-11-12T07:48:07,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741965_1141 (size=30955) 2024-11-12T07:48:07,451 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/7215861668b74c09a525513166a85c08 2024-11-12T07:48:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T07:48:07,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ba1a4211157044fab2158a789b681d1c is 50, key is test_row_0/B:col10/1731397686388/Put/seqid=0 2024-11-12T07:48:07,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741966_1142 (size=12001) 2024-11-12T07:48:07,490 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ba1a4211157044fab2158a789b681d1c 2024-11-12T07:48:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/9196e3a1e6284bf4b7f2b5b911b39fc2 is 50, key is test_row_0/C:col10/1731397686388/Put/seqid=0 2024-11-12T07:48:07,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741967_1143 (size=12001) 2024-11-12T07:48:07,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:07,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397747559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397747561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397747562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397747566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397747566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397747679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397747681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397747681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397747681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397747682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397747888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397747888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397747888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397747888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:07,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397747888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:07,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/9196e3a1e6284bf4b7f2b5b911b39fc2 2024-11-12T07:48:07,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/7215861668b74c09a525513166a85c08 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/7215861668b74c09a525513166a85c08 2024-11-12T07:48:07,983 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/7215861668b74c09a525513166a85c08, entries=150, sequenceid=37, filesize=30.2 K 2024-11-12T07:48:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ba1a4211157044fab2158a789b681d1c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba1a4211157044fab2158a789b681d1c 2024-11-12T07:48:08,001 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba1a4211157044fab2158a789b681d1c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T07:48:08,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/9196e3a1e6284bf4b7f2b5b911b39fc2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/9196e3a1e6284bf4b7f2b5b911b39fc2 2024-11-12T07:48:08,012 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/9196e3a1e6284bf4b7f2b5b911b39fc2, entries=150, sequenceid=37, filesize=11.7 K 2024-11-12T07:48:08,014 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 299b057d313257fac3b10771c8ca0f20 in 621ms, sequenceid=37, compaction requested=false 2024-11-12T07:48:08,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:08,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:08,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-12T07:48:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-12T07:48:08,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-12T07:48:08,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7040 sec 2024-11-12T07:48:08,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7150 sec 2024-11-12T07:48:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:08,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:48:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:08,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:08,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127cf9ec4960f8440c87c90f7cb5d386c2_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397688199/Put/seqid=0 2024-11-12T07:48:08,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397748224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397748226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397748228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397748232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397748234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741968_1144 (size=12154) 2024-11-12T07:48:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397748338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397748338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397748338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397748338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397748338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T07:48:08,468 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:08,468 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-12T07:48:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T07:48:08,470 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:08,471 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:08,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397748548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397748548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397748559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397748559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397748559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T07:48:08,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T07:48:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:08,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:08,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:08,642 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:08,650 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127cf9ec4960f8440c87c90f7cb5d386c2_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127cf9ec4960f8440c87c90f7cb5d386c2_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:08,652 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/890cab92fd5641dd8b33ee20711c99f1, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:08,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/890cab92fd5641dd8b33ee20711c99f1 is 175, key is test_row_0/A:col10/1731397688199/Put/seqid=0 2024-11-12T07:48:08,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741969_1145 (size=30955) 2024-11-12T07:48:08,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T07:48:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:08,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:08,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T07:48:08,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397748857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397748858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397748868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397748868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397748868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:08,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:08,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T07:48:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:08,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/890cab92fd5641dd8b33ee20711c99f1 2024-11-12T07:48:09,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/8818d91e5ebd48e785dbd3a27220df10 is 50, key is test_row_0/B:col10/1731397688199/Put/seqid=0 2024-11-12T07:48:09,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T07:48:09,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:09,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741970_1146 (size=12001) 2024-11-12T07:48:09,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/8818d91e5ebd48e785dbd3a27220df10 2024-11-12T07:48:09,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/778268ae7a3246d2bcc63d93f7baa470 is 50, key is test_row_0/C:col10/1731397688199/Put/seqid=0 2024-11-12T07:48:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T07:48:09,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741971_1147 (size=12001) 2024-11-12T07:48:09,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T07:48:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397749368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:09,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397749368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:09,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:09,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397749377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:09,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397749378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:09,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397749379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:09,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T07:48:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:09,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/778268ae7a3246d2bcc63d93f7baa470 2024-11-12T07:48:09,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/890cab92fd5641dd8b33ee20711c99f1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/890cab92fd5641dd8b33ee20711c99f1 2024-11-12T07:48:09,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/890cab92fd5641dd8b33ee20711c99f1, entries=150, sequenceid=51, filesize=30.2 K 2024-11-12T07:48:09,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/8818d91e5ebd48e785dbd3a27220df10 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/8818d91e5ebd48e785dbd3a27220df10 2024-11-12T07:48:09,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/8818d91e5ebd48e785dbd3a27220df10, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T07:48:09,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/778268ae7a3246d2bcc63d93f7baa470 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/778268ae7a3246d2bcc63d93f7baa470 2024-11-12T07:48:09,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/778268ae7a3246d2bcc63d93f7baa470, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T07:48:09,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 299b057d313257fac3b10771c8ca0f20 in 1327ms, sequenceid=51, compaction requested=true 2024-11-12T07:48:09,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:09,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:09,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:09,529 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:09,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:09,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:09,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:09,529 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:09,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:09,530 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:09,530 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:09,530 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/B is initiating minor compaction (all files) 2024-11-12T07:48:09,530 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/A is initiating minor compaction (all files) 2024-11-12T07:48:09,530 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/B in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,530 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/A in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,530 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4531087f146e473691874f4ec27314fd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba1a4211157044fab2158a789b681d1c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/8818d91e5ebd48e785dbd3a27220df10] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=35.2 K 2024-11-12T07:48:09,530 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d06569a02fd949609e62d5e23c6309de, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/7215861668b74c09a525513166a85c08, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/890cab92fd5641dd8b33ee20711c99f1] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=115.9 K 2024-11-12T07:48:09,530 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,531 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 4531087f146e473691874f4ec27314fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731397686327 2024-11-12T07:48:09,531 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d06569a02fd949609e62d5e23c6309de, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/7215861668b74c09a525513166a85c08, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/890cab92fd5641dd8b33ee20711c99f1] 2024-11-12T07:48:09,531 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ba1a4211157044fab2158a789b681d1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731397686367 2024-11-12T07:48:09,531 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d06569a02fd949609e62d5e23c6309de, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731397686327 2024-11-12T07:48:09,531 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8818d91e5ebd48e785dbd3a27220df10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731397687557 2024-11-12T07:48:09,531 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7215861668b74c09a525513166a85c08, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731397686367 2024-11-12T07:48:09,532 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890cab92fd5641dd8b33ee20711c99f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731397687557 2024-11-12T07:48:09,537 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:09,538 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#B#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:09,538 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ba630b74d9d946c691d1c6ba875a776b is 50, key is test_row_0/B:col10/1731397688199/Put/seqid=0 2024-11-12T07:48:09,540 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411122aafc92999a242f1b61fa334bc46a742_299b057d313257fac3b10771c8ca0f20 store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:09,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T07:48:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,547 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:48:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:09,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:09,554 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411122aafc92999a242f1b61fa334bc46a742_299b057d313257fac3b10771c8ca0f20, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:09,555 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122aafc92999a242f1b61fa334bc46a742_299b057d313257fac3b10771c8ca0f20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:09,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741972_1148 (size=12104) 2024-11-12T07:48:09,568 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ba630b74d9d946c691d1c6ba875a776b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba630b74d9d946c691d1c6ba875a776b 2024-11-12T07:48:09,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111230e57fab68df488b87bc7fde0b127b45_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397688231/Put/seqid=0 2024-11-12T07:48:09,579 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/B of 299b057d313257fac3b10771c8ca0f20 into ba630b74d9d946c691d1c6ba875a776b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:09,579 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:09,579 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/B, priority=13, startTime=1731397689529; duration=0sec 2024-11-12T07:48:09,579 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:09,579 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:B 2024-11-12T07:48:09,579 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:09,581 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:09,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741973_1149 (size=4469) 2024-11-12T07:48:09,582 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/C is initiating minor compaction (all files) 2024-11-12T07:48:09,582 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/C in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:09,582 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d5d456b5595a44d89e04b3c94801190c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/9196e3a1e6284bf4b7f2b5b911b39fc2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/778268ae7a3246d2bcc63d93f7baa470] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=35.2 K 2024-11-12T07:48:09,584 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d5d456b5595a44d89e04b3c94801190c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731397686327 2024-11-12T07:48:09,584 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#A#compaction#130 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:09,584 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9196e3a1e6284bf4b7f2b5b911b39fc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731397686367 2024-11-12T07:48:09,585 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 778268ae7a3246d2bcc63d93f7baa470, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731397687557 2024-11-12T07:48:09,585 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/c44c05857f054f4fad618ab83b8a3bb9 is 175, key is test_row_0/A:col10/1731397688199/Put/seqid=0 2024-11-12T07:48:09,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741974_1150 (size=12154) 2024-11-12T07:48:09,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741975_1151 (size=31058) 2024-11-12T07:48:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T07:48:09,609 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/c44c05857f054f4fad618ab83b8a3bb9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c44c05857f054f4fad618ab83b8a3bb9 2024-11-12T07:48:09,617 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/A of 299b057d313257fac3b10771c8ca0f20 into c44c05857f054f4fad618ab83b8a3bb9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:09,617 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:09,617 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/A, priority=13, startTime=1731397689529; duration=0sec 2024-11-12T07:48:09,617 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:09,617 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:A 2024-11-12T07:48:09,621 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:09,622 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/35608baa4ca449eabb7c4f8768ebf11a is 50, key is test_row_0/C:col10/1731397688199/Put/seqid=0 2024-11-12T07:48:09,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741976_1152 (size=12104) 2024-11-12T07:48:09,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:09,992 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111230e57fab68df488b87bc7fde0b127b45_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111230e57fab68df488b87bc7fde0b127b45_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:09,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/8f2db8af5a464fd58c45d030604fb70b, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:09,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/8f2db8af5a464fd58c45d030604fb70b is 175, key is test_row_0/A:col10/1731397688231/Put/seqid=0 2024-11-12T07:48:10,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741977_1153 (size=30955) 2024-11-12T07:48:10,004 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/8f2db8af5a464fd58c45d030604fb70b 2024-11-12T07:48:10,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/a37260779d894db2901ad2f79e4ee32a is 50, key is test_row_0/B:col10/1731397688231/Put/seqid=0 2024-11-12T07:48:10,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741978_1154 (size=12001) 2024-11-12T07:48:10,054 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/35608baa4ca449eabb7c4f8768ebf11a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/35608baa4ca449eabb7c4f8768ebf11a 2024-11-12T07:48:10,062 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/C of 299b057d313257fac3b10771c8ca0f20 into 35608baa4ca449eabb7c4f8768ebf11a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:10,062 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:10,062 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/C, priority=13, startTime=1731397689529; duration=0sec 2024-11-12T07:48:10,062 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:10,062 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:C 2024-11-12T07:48:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:10,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:10,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397750402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397750403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397750403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397750404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397750404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,433 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/a37260779d894db2901ad2f79e4ee32a 2024-11-12T07:48:10,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/c3747c43ebc14cf09ac8878360bef6f5 is 50, key is test_row_0/C:col10/1731397688231/Put/seqid=0 2024-11-12T07:48:10,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741979_1155 (size=12001) 2024-11-12T07:48:10,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397750518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397750518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397750518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397750519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397750519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T07:48:10,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397750728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397750728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397750728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397750729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397750729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:10,887 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/c3747c43ebc14cf09ac8878360bef6f5 2024-11-12T07:48:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/8f2db8af5a464fd58c45d030604fb70b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8f2db8af5a464fd58c45d030604fb70b 2024-11-12T07:48:10,898 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8f2db8af5a464fd58c45d030604fb70b, entries=150, sequenceid=69, filesize=30.2 K 2024-11-12T07:48:10,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/a37260779d894db2901ad2f79e4ee32a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/a37260779d894db2901ad2f79e4ee32a 2024-11-12T07:48:10,904 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/a37260779d894db2901ad2f79e4ee32a, entries=150, sequenceid=69, filesize=11.7 K 2024-11-12T07:48:10,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/c3747c43ebc14cf09ac8878360bef6f5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/c3747c43ebc14cf09ac8878360bef6f5 2024-11-12T07:48:10,909 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/c3747c43ebc14cf09ac8878360bef6f5, entries=150, sequenceid=69, filesize=11.7 K 2024-11-12T07:48:10,910 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 299b057d313257fac3b10771c8ca0f20 in 1363ms, sequenceid=69, compaction requested=false 2024-11-12T07:48:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-12T07:48:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-12T07:48:10,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T07:48:10,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4410 sec 2024-11-12T07:48:10,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.4450 sec 2024-11-12T07:48:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:11,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T07:48:11,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:11,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:11,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:11,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:11,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:11,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:11,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112eb9b21b6b10b4000981060637b0b6c1d_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397691038/Put/seqid=0 2024-11-12T07:48:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397751052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397751053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397751054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397751055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397751056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741980_1156 (size=12154) 2024-11-12T07:48:11,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397751168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397751168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397751168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397751168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397751168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397751377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397751377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397751378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397751378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397751378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,465 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:11,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112eb9b21b6b10b4000981060637b0b6c1d_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eb9b21b6b10b4000981060637b0b6c1d_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:11,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/e9395082b741470789163eaea73ff65a, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:11,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/e9395082b741470789163eaea73ff65a is 175, key is test_row_0/A:col10/1731397691038/Put/seqid=0 2024-11-12T07:48:11,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741981_1157 (size=30955) 2024-11-12T07:48:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397751688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397751688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397751688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397751688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397751688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:11,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:11,728 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:48:11,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/e9395082b741470789163eaea73ff65a 2024-11-12T07:48:11,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/5fcd08576c47443fa05e13e98a53d26f is 50, key is test_row_0/B:col10/1731397691038/Put/seqid=0 2024-11-12T07:48:11,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741982_1158 (size=12001) 2024-11-12T07:48:12,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397752197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:12,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397752198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:12,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397752198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:12,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397752208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397752208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:12,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:12,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/5fcd08576c47443fa05e13e98a53d26f 2024-11-12T07:48:12,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/e070f43c3c994181a841f52d2dccac07 is 50, key is test_row_0/C:col10/1731397691038/Put/seqid=0 2024-11-12T07:48:12,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741983_1159 (size=12001) 2024-11-12T07:48:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T07:48:12,648 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:12,649 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-12T07:48:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T07:48:12,651 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:12,651 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:12,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:12,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/e070f43c3c994181a841f52d2dccac07 2024-11-12T07:48:12,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/e9395082b741470789163eaea73ff65a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/e9395082b741470789163eaea73ff65a 2024-11-12T07:48:12,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/e9395082b741470789163eaea73ff65a, entries=150, sequenceid=89, filesize=30.2 K 2024-11-12T07:48:12,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/5fcd08576c47443fa05e13e98a53d26f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5fcd08576c47443fa05e13e98a53d26f 2024-11-12T07:48:12,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5fcd08576c47443fa05e13e98a53d26f, entries=150, sequenceid=89, filesize=11.7 K 2024-11-12T07:48:12,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/e070f43c3c994181a841f52d2dccac07 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/e070f43c3c994181a841f52d2dccac07 2024-11-12T07:48:12,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/e070f43c3c994181a841f52d2dccac07, entries=150, sequenceid=89, filesize=11.7 K 2024-11-12T07:48:12,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 299b057d313257fac3b10771c8ca0f20 in 1684ms, sequenceid=89, compaction requested=true 2024-11-12T07:48:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:12,724 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:12,724 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:12,725 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:12,725 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:12,725 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/A is initiating minor compaction (all files) 2024-11-12T07:48:12,725 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/B is initiating minor compaction (all files) 2024-11-12T07:48:12,725 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/A in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:12,725 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/B in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:12,725 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba630b74d9d946c691d1c6ba875a776b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/a37260779d894db2901ad2f79e4ee32a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5fcd08576c47443fa05e13e98a53d26f] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=35.3 K 2024-11-12T07:48:12,725 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c44c05857f054f4fad618ab83b8a3bb9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8f2db8af5a464fd58c45d030604fb70b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/e9395082b741470789163eaea73ff65a] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=90.8 K 2024-11-12T07:48:12,725 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:12,725 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c44c05857f054f4fad618ab83b8a3bb9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8f2db8af5a464fd58c45d030604fb70b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/e9395082b741470789163eaea73ff65a] 2024-11-12T07:48:12,726 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ba630b74d9d946c691d1c6ba875a776b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731397687557 2024-11-12T07:48:12,726 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting c44c05857f054f4fad618ab83b8a3bb9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731397687557 2024-11-12T07:48:12,726 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting a37260779d894db2901ad2f79e4ee32a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397688219 2024-11-12T07:48:12,726 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f2db8af5a464fd58c45d030604fb70b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397688219 2024-11-12T07:48:12,726 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fcd08576c47443fa05e13e98a53d26f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731397690400 2024-11-12T07:48:12,726 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9395082b741470789163eaea73ff65a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731397690400 2024-11-12T07:48:12,735 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:12,738 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ddca3fa235f74102bd78d33140d970c2_299b057d313257fac3b10771c8ca0f20 store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:12,739 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:12,739 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/69b311d273e745b68e536b328dcfa1eb is 50, key is test_row_0/B:col10/1731397691038/Put/seqid=0 2024-11-12T07:48:12,741 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ddca3fa235f74102bd78d33140d970c2_299b057d313257fac3b10771c8ca0f20, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:12,741 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ddca3fa235f74102bd78d33140d970c2_299b057d313257fac3b10771c8ca0f20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:12,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741984_1160 (size=4469) 2024-11-12T07:48:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T07:48:12,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741985_1161 (size=12207) 2024-11-12T07:48:12,768 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/69b311d273e745b68e536b328dcfa1eb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/69b311d273e745b68e536b328dcfa1eb 2024-11-12T07:48:12,776 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/B of 299b057d313257fac3b10771c8ca0f20 into 69b311d273e745b68e536b328dcfa1eb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:12,776 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:12,777 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/B, priority=13, startTime=1731397692724; duration=0sec 2024-11-12T07:48:12,777 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:12,777 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:B 2024-11-12T07:48:12,777 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:12,779 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:12,779 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/C is initiating minor compaction (all files) 2024-11-12T07:48:12,779 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/C in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:12,779 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/35608baa4ca449eabb7c4f8768ebf11a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/c3747c43ebc14cf09ac8878360bef6f5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/e070f43c3c994181a841f52d2dccac07] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=35.3 K 2024-11-12T07:48:12,779 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 35608baa4ca449eabb7c4f8768ebf11a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731397687557 2024-11-12T07:48:12,780 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c3747c43ebc14cf09ac8878360bef6f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731397688219 2024-11-12T07:48:12,781 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e070f43c3c994181a841f52d2dccac07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731397690400 2024-11-12T07:48:12,790 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:12,791 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d7a290f01b0241d78f0cd8d1f6348796 is 50, key is test_row_0/C:col10/1731397691038/Put/seqid=0 2024-11-12T07:48:12,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741986_1162 (size=12207) 2024-11-12T07:48:12,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-12T07:48:12,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:12,804 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T07:48:12,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:12,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:12,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:12,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:12,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:12,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:12,807 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d7a290f01b0241d78f0cd8d1f6348796 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d7a290f01b0241d78f0cd8d1f6348796 2024-11-12T07:48:12,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a0246e30a4954b3085fa3b28924aaedf_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397691052/Put/seqid=0 2024-11-12T07:48:12,819 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/C of 299b057d313257fac3b10771c8ca0f20 into d7a290f01b0241d78f0cd8d1f6348796(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:12,819 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:12,819 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/C, priority=13, startTime=1731397692724; duration=0sec 2024-11-12T07:48:12,819 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:12,819 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:C 2024-11-12T07:48:12,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741987_1163 (size=12154) 2024-11-12T07:48:12,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:12,826 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a0246e30a4954b3085fa3b28924aaedf_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a0246e30a4954b3085fa3b28924aaedf_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:12,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/8d6ebd4e1be74db59d7c7cafb596b9fb, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:12,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/8d6ebd4e1be74db59d7c7cafb596b9fb is 175, key is test_row_0/A:col10/1731397691052/Put/seqid=0 2024-11-12T07:48:12,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741988_1164 (size=30955) 2024-11-12T07:48:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T07:48:13,189 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:13,190 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/ef064779c962485abeb67c3aebb71913 is 175, key is test_row_0/A:col10/1731397691038/Put/seqid=0 2024-11-12T07:48:13,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741989_1165 (size=31161) 2024-11-12T07:48:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:13,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:13,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,238 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/8d6ebd4e1be74db59d7c7cafb596b9fb 2024-11-12T07:48:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397753229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397753230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397753232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397753233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397753238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/27746e1378a142b0b04dd9a1bb723b27 is 50, key is test_row_0/B:col10/1731397691052/Put/seqid=0 2024-11-12T07:48:13,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741990_1166 (size=12001) 2024-11-12T07:48:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T07:48:13,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397753347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397753347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397753348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397753348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397753348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397753558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397753558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397753558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397753558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397753558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,602 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/ef064779c962485abeb67c3aebb71913 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/ef064779c962485abeb67c3aebb71913 2024-11-12T07:48:13,607 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/A of 299b057d313257fac3b10771c8ca0f20 into ef064779c962485abeb67c3aebb71913(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:13,607 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:13,607 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/A, priority=13, startTime=1731397692724; duration=0sec 2024-11-12T07:48:13,607 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:13,607 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:A 2024-11-12T07:48:13,662 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/27746e1378a142b0b04dd9a1bb723b27 2024-11-12T07:48:13,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/39f0f19c475f46db953f552b58a804ab is 50, key is test_row_0/C:col10/1731397691052/Put/seqid=0 2024-11-12T07:48:13,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741991_1167 (size=12001) 2024-11-12T07:48:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T07:48:13,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397753868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397753868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397753868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397753868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397753868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,076 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/39f0f19c475f46db953f552b58a804ab 2024-11-12T07:48:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/8d6ebd4e1be74db59d7c7cafb596b9fb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8d6ebd4e1be74db59d7c7cafb596b9fb 2024-11-12T07:48:14,085 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8d6ebd4e1be74db59d7c7cafb596b9fb, entries=150, sequenceid=105, filesize=30.2 K 2024-11-12T07:48:14,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/27746e1378a142b0b04dd9a1bb723b27 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/27746e1378a142b0b04dd9a1bb723b27 2024-11-12T07:48:14,091 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/27746e1378a142b0b04dd9a1bb723b27, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T07:48:14,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/39f0f19c475f46db953f552b58a804ab as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/39f0f19c475f46db953f552b58a804ab 2024-11-12T07:48:14,096 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/39f0f19c475f46db953f552b58a804ab, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T07:48:14,097 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 299b057d313257fac3b10771c8ca0f20 in 1293ms, sequenceid=105, compaction requested=false 2024-11-12T07:48:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-12T07:48:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-12T07:48:14,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T07:48:14,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-11-12T07:48:14,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4510 sec 2024-11-12T07:48:14,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:14,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:48:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:14,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127d23fa86319d4b17af2f4c7bb092515f_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397694378/Put/seqid=0 2024-11-12T07:48:14,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741992_1168 (size=12154) 2024-11-12T07:48:14,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397754387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397754391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397754391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397754392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397754393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397754498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397754498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397754498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397754507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397754507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397754708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397754708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397754708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397754717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397754718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:14,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:14,795 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127d23fa86319d4b17af2f4c7bb092515f_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127d23fa86319d4b17af2f4c7bb092515f_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:14,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d00ca1391dd34c5f8c30a1a784ec7132, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:14,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d00ca1391dd34c5f8c30a1a784ec7132 is 175, key is test_row_0/A:col10/1731397694378/Put/seqid=0 2024-11-12T07:48:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T07:48:14,798 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:14,799 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-12T07:48:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T07:48:14,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:14,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741993_1169 (size=30955) 2024-11-12T07:48:14,803 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:14,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:14,803 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d00ca1391dd34c5f8c30a1a784ec7132 2024-11-12T07:48:14,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/3d19013a89a44c058c920e14c828d19a is 50, key is test_row_0/B:col10/1731397694378/Put/seqid=0 2024-11-12T07:48:14,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741994_1170 (size=12001) 2024-11-12T07:48:14,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/3d19013a89a44c058c920e14c828d19a 2024-11-12T07:48:14,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/5de757839dd44e29a1ce40c701b164d0 is 50, key is test_row_0/C:col10/1731397694378/Put/seqid=0 2024-11-12T07:48:14,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741995_1171 (size=12001) 2024-11-12T07:48:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T07:48:14,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T07:48:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:14,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:15,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397755018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397755018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397755028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397755028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397755028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T07:48:15,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:15,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:15,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:15,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:15,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T07:48:15,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/5de757839dd44e29a1ce40c701b164d0 2024-11-12T07:48:15,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d00ca1391dd34c5f8c30a1a784ec7132 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d00ca1391dd34c5f8c30a1a784ec7132 2024-11-12T07:48:15,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d00ca1391dd34c5f8c30a1a784ec7132, entries=150, sequenceid=125, filesize=30.2 K 2024-11-12T07:48:15,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/3d19013a89a44c058c920e14c828d19a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/3d19013a89a44c058c920e14c828d19a 2024-11-12T07:48:15,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/3d19013a89a44c058c920e14c828d19a, entries=150, sequenceid=125, filesize=11.7 K 2024-11-12T07:48:15,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/5de757839dd44e29a1ce40c701b164d0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/5de757839dd44e29a1ce40c701b164d0 2024-11-12T07:48:15,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/5de757839dd44e29a1ce40c701b164d0, entries=150, sequenceid=125, filesize=11.7 K 2024-11-12T07:48:15,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 299b057d313257fac3b10771c8ca0f20 in 877ms, sequenceid=125, compaction requested=true 2024-11-12T07:48:15,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:15,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:15,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:15,257 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:15,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:15,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:15,257 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:15,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:15,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:15,258 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93071 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:15,258 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:15,258 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/A is initiating minor compaction (all files) 2024-11-12T07:48:15,258 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/B is initiating minor compaction (all files) 2024-11-12T07:48:15,258 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/B in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:15,258 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/A in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:15,258 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/ef064779c962485abeb67c3aebb71913, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8d6ebd4e1be74db59d7c7cafb596b9fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d00ca1391dd34c5f8c30a1a784ec7132] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=90.9 K 2024-11-12T07:48:15,258 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/69b311d273e745b68e536b328dcfa1eb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/27746e1378a142b0b04dd9a1bb723b27, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/3d19013a89a44c058c920e14c828d19a] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=35.4 K 2024-11-12T07:48:15,258 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:15,258 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/ef064779c962485abeb67c3aebb71913, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8d6ebd4e1be74db59d7c7cafb596b9fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d00ca1391dd34c5f8c30a1a784ec7132] 2024-11-12T07:48:15,259 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b311d273e745b68e536b328dcfa1eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731397690400 2024-11-12T07:48:15,259 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef064779c962485abeb67c3aebb71913, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731397690400 2024-11-12T07:48:15,259 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 27746e1378a142b0b04dd9a1bb723b27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731397691049 2024-11-12T07:48:15,259 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d6ebd4e1be74db59d7c7cafb596b9fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731397691049 2024-11-12T07:48:15,259 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d19013a89a44c058c920e14c828d19a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731397693227 2024-11-12T07:48:15,259 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d00ca1391dd34c5f8c30a1a784ec7132, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731397693227 2024-11-12T07:48:15,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-12T07:48:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:15,262 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:48:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:15,268 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:15,268 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/33f1012808db46879cc9ce5aaf5c1cc7 is 50, key is test_row_0/B:col10/1731397694378/Put/seqid=0 2024-11-12T07:48:15,270 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:15,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b504018378de43aca7502bdc615f0d82_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397694390/Put/seqid=0 2024-11-12T07:48:15,282 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112a8d349e2e2eb498aa26e0ecde342aec3_299b057d313257fac3b10771c8ca0f20 store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:15,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741996_1172 (size=12309) 2024-11-12T07:48:15,284 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112a8d349e2e2eb498aa26e0ecde342aec3_299b057d313257fac3b10771c8ca0f20, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:15,285 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a8d349e2e2eb498aa26e0ecde342aec3_299b057d313257fac3b10771c8ca0f20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:15,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741997_1173 (size=12304) 2024-11-12T07:48:15,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:15,294 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/33f1012808db46879cc9ce5aaf5c1cc7 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/33f1012808db46879cc9ce5aaf5c1cc7 2024-11-12T07:48:15,297 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b504018378de43aca7502bdc615f0d82_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b504018378de43aca7502bdc615f0d82_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/3129c4d83bf64248bf75370701437367, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:15,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/3129c4d83bf64248bf75370701437367 is 175, key is test_row_0/A:col10/1731397694390/Put/seqid=0 2024-11-12T07:48:15,305 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/B of 299b057d313257fac3b10771c8ca0f20 into 33f1012808db46879cc9ce5aaf5c1cc7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:15,305 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:15,305 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/B, priority=13, startTime=1731397695257; duration=0sec 2024-11-12T07:48:15,305 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:15,305 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:B 2024-11-12T07:48:15,305 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:15,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741998_1174 (size=4469) 2024-11-12T07:48:15,307 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#A#compaction#148 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:15,308 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:15,308 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/C is initiating minor compaction (all files) 2024-11-12T07:48:15,308 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/C in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:15,308 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/f2828ea19dd447dfbcac36f4a39e69a7 is 175, key is test_row_0/A:col10/1731397694378/Put/seqid=0 2024-11-12T07:48:15,308 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d7a290f01b0241d78f0cd8d1f6348796, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/39f0f19c475f46db953f552b58a804ab, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/5de757839dd44e29a1ce40c701b164d0] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=35.4 K 2024-11-12T07:48:15,309 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a290f01b0241d78f0cd8d1f6348796, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731397690400 2024-11-12T07:48:15,309 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 39f0f19c475f46db953f552b58a804ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731397691049 2024-11-12T07:48:15,310 INFO [master/f9fc96254ef8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T07:48:15,310 INFO [master/f9fc96254ef8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T07:48:15,313 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de757839dd44e29a1ce40c701b164d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731397693227 2024-11-12T07:48:15,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741999_1175 (size=31105) 2024-11-12T07:48:15,322 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/3129c4d83bf64248bf75370701437367 2024-11-12T07:48:15,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742000_1176 (size=31263) 2024-11-12T07:48:15,332 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#C#compaction#150 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:15,333 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/8d320e0f206c4773adc6b9aaa4bad1ce is 50, key is test_row_0/C:col10/1731397694378/Put/seqid=0 2024-11-12T07:48:15,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742001_1177 (size=12309) 2024-11-12T07:48:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/e777a1fcf0c643c389edca032bc14fd1 is 50, key is test_row_0/B:col10/1731397694390/Put/seqid=0 2024-11-12T07:48:15,352 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/8d320e0f206c4773adc6b9aaa4bad1ce as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8d320e0f206c4773adc6b9aaa4bad1ce 2024-11-12T07:48:15,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742002_1178 (size=12151) 2024-11-12T07:48:15,357 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/e777a1fcf0c643c389edca032bc14fd1 2024-11-12T07:48:15,359 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/C of 299b057d313257fac3b10771c8ca0f20 into 8d320e0f206c4773adc6b9aaa4bad1ce(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:15,359 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:15,359 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/C, priority=13, startTime=1731397695257; duration=0sec 2024-11-12T07:48:15,359 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:15,359 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:C 2024-11-12T07:48:15,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/19da1e97d21d4f57bfae6566a7769fc3 is 50, key is test_row_0/C:col10/1731397694390/Put/seqid=0 2024-11-12T07:48:15,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742003_1179 (size=12151) 2024-11-12T07:48:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T07:48:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:15,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397755548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397755548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397755549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397755550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397755554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397755658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397755658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397755658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,738 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/f2828ea19dd447dfbcac36f4a39e69a7 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f2828ea19dd447dfbcac36f4a39e69a7 2024-11-12T07:48:15,743 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/A of 299b057d313257fac3b10771c8ca0f20 into f2828ea19dd447dfbcac36f4a39e69a7(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:15,743 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:15,743 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/A, priority=13, startTime=1731397695257; duration=0sec 2024-11-12T07:48:15,743 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:15,743 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:A 2024-11-12T07:48:15,778 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/19da1e97d21d4f57bfae6566a7769fc3 2024-11-12T07:48:15,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/3129c4d83bf64248bf75370701437367 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/3129c4d83bf64248bf75370701437367 2024-11-12T07:48:15,788 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/3129c4d83bf64248bf75370701437367, entries=150, sequenceid=139, filesize=30.4 K 2024-11-12T07:48:15,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/e777a1fcf0c643c389edca032bc14fd1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/e777a1fcf0c643c389edca032bc14fd1 2024-11-12T07:48:15,797 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/e777a1fcf0c643c389edca032bc14fd1, entries=150, sequenceid=139, filesize=11.9 K 2024-11-12T07:48:15,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/19da1e97d21d4f57bfae6566a7769fc3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/19da1e97d21d4f57bfae6566a7769fc3 2024-11-12T07:48:15,805 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/19da1e97d21d4f57bfae6566a7769fc3, entries=150, sequenceid=139, filesize=11.9 K 2024-11-12T07:48:15,806 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 299b057d313257fac3b10771c8ca0f20 in 544ms, sequenceid=139, compaction requested=false 2024-11-12T07:48:15,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:15,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:15,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-12T07:48:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-12T07:48:15,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-12T07:48:15,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-12T07:48:15,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.0110 sec 2024-11-12T07:48:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:15,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-12T07:48:15,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397695548, max timestamp=1731397695871Num uniques 859; 2024-11-12T07:48:15,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:48:15,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397695548, max timestamp=1731397695871Num uniques 859; 2024-11-12T07:48:15,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:48:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:15,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:15,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d40cd1e6cd844eb4bdbdc97c3dc9021c_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397695548/Put/seqid=0 2024-11-12T07:48:15,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397755922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397695906, max timestamp=1731397695924Num uniques 859; 2024-11-12T07:48:15,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397695906, max timestamp=1731397695924Num uniques 859; 2024-11-12T07:48:15,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:48:15,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:48:15,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397695906, max timestamp=1731397695924Num uniques 859; 2024-11-12T07:48:15,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:48:15,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742004_1180 (size=14794) 2024-11-12T07:48:15,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-12T07:48:15,939 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:15,939 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-12T07:48:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T07:48:15,941 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:15,946 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:15,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:15,950 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d40cd1e6cd844eb4bdbdc97c3dc9021c_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d40cd1e6cd844eb4bdbdc97c3dc9021c_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:15,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d84c7ce189084ef5935ee6dc4473ef61, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:15,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d84c7ce189084ef5935ee6dc4473ef61 is 175, key is test_row_0/A:col10/1731397695548/Put/seqid=0 2024-11-12T07:48:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397755986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397755987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742005_1181 (size=39749) 2024-11-12T07:48:15,997 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d84c7ce189084ef5935ee6dc4473ef61 2024-11-12T07:48:16,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ce552e4071f64c0c9d3328ed265c011e is 50, key is test_row_0/B:col10/1731397695548/Put/seqid=0 2024-11-12T07:48:16,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742006_1182 (size=12151) 2024-11-12T07:48:16,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397756038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T07:48:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397756098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397756098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T07:48:16,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397756248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T07:48:16,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T07:48:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397756308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397756308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T07:48:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:16,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ce552e4071f64c0c9d3328ed265c011e 2024-11-12T07:48:16,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/4e92d82ffcb6451cbbf3fb4dab1da527 is 50, key is test_row_0/C:col10/1731397695548/Put/seqid=0 2024-11-12T07:48:16,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742007_1183 (size=12151) 2024-11-12T07:48:16,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397756558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T07:48:16,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:16,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T07:48:16,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397756567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397756568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,572 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_0' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:16,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,572 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:16,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397756618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397756618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:16,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:16,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T07:48:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/4e92d82ffcb6451cbbf3fb4dab1da527 2024-11-12T07:48:16,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/d84c7ce189084ef5935ee6dc4473ef61 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d84c7ce189084ef5935ee6dc4473ef61 2024-11-12T07:48:16,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d84c7ce189084ef5935ee6dc4473ef61, entries=200, sequenceid=163, filesize=38.8 K 2024-11-12T07:48:16,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ce552e4071f64c0c9d3328ed265c011e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce552e4071f64c0c9d3328ed265c011e 2024-11-12T07:48:16,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce552e4071f64c0c9d3328ed265c011e, entries=150, sequenceid=163, filesize=11.9 K 2024-11-12T07:48:16,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T07:48:16,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/4e92d82ffcb6451cbbf3fb4dab1da527 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4e92d82ffcb6451cbbf3fb4dab1da527 2024-11-12T07:48:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:16,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4e92d82ffcb6451cbbf3fb4dab1da527, entries=150, sequenceid=163, filesize=11.9 K 2024-11-12T07:48:16,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 299b057d313257fac3b10771c8ca0f20 in 1005ms, sequenceid=163, compaction requested=true 2024-11-12T07:48:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:16,879 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:16,879 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:16,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:16,880 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:16,880 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/A is initiating minor compaction (all files) 2024-11-12T07:48:16,881 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/A in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,881 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f2828ea19dd447dfbcac36f4a39e69a7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/3129c4d83bf64248bf75370701437367, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d84c7ce189084ef5935ee6dc4473ef61] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=99.7 K 2024-11-12T07:48:16,881 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,881 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f2828ea19dd447dfbcac36f4a39e69a7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/3129c4d83bf64248bf75370701437367, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d84c7ce189084ef5935ee6dc4473ef61] 2024-11-12T07:48:16,881 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:16,881 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/B is initiating minor compaction (all files) 2024-11-12T07:48:16,881 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/B in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:16,882 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/33f1012808db46879cc9ce5aaf5c1cc7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/e777a1fcf0c643c389edca032bc14fd1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce552e4071f64c0c9d3328ed265c011e] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=35.8 K 2024-11-12T07:48:16,882 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2828ea19dd447dfbcac36f4a39e69a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731397693227 2024-11-12T07:48:16,882 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 33f1012808db46879cc9ce5aaf5c1cc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731397693227 2024-11-12T07:48:16,882 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3129c4d83bf64248bf75370701437367, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731397694385 2024-11-12T07:48:16,882 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e777a1fcf0c643c389edca032bc14fd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731397694385 2024-11-12T07:48:16,882 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d84c7ce189084ef5935ee6dc4473ef61, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731397695548 2024-11-12T07:48:16,883 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ce552e4071f64c0c9d3328ed265c011e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731397695548 2024-11-12T07:48:16,889 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:16,903 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111231639b7be2814c4cb6010b688a2ff740_299b057d313257fac3b10771c8ca0f20 store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:16,905 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:16,906 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ce09371cb1464a3e9761e48ea8022c7b is 50, key is test_row_0/B:col10/1731397695548/Put/seqid=0 2024-11-12T07:48:16,906 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111231639b7be2814c4cb6010b688a2ff740_299b057d313257fac3b10771c8ca0f20, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:16,906 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111231639b7be2814c4cb6010b688a2ff740_299b057d313257fac3b10771c8ca0f20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:16,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742008_1184 (size=12561) 2024-11-12T07:48:16,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742009_1185 (size=4469) 2024-11-12T07:48:17,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-12T07:48:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:17,023 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-12T07:48:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c15cdbc1df4240e2be3055bfe02af5f7_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397695985/Put/seqid=0 2024-11-12T07:48:17,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742010_1186 (size=12304) 2024-11-12T07:48:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:17,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T07:48:17,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397757101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:17,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397757137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397757137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:17,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397757218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:17,318 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:17,319 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/db51becab35043858815203259c2182f is 175, key is test_row_0/A:col10/1731397695548/Put/seqid=0 2024-11-12T07:48:17,321 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/ce09371cb1464a3e9761e48ea8022c7b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce09371cb1464a3e9761e48ea8022c7b 2024-11-12T07:48:17,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742011_1187 (size=31515) 2024-11-12T07:48:17,327 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/db51becab35043858815203259c2182f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/db51becab35043858815203259c2182f 2024-11-12T07:48:17,328 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/B of 299b057d313257fac3b10771c8ca0f20 into ce09371cb1464a3e9761e48ea8022c7b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:17,328 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:17,329 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/B, priority=13, startTime=1731397696879; duration=0sec 2024-11-12T07:48:17,329 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:17,329 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:B 2024-11-12T07:48:17,329 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:17,330 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:17,331 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/C is initiating minor compaction (all files) 2024-11-12T07:48:17,331 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/C in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:17,331 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8d320e0f206c4773adc6b9aaa4bad1ce, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/19da1e97d21d4f57bfae6566a7769fc3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4e92d82ffcb6451cbbf3fb4dab1da527] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=35.8 K 2024-11-12T07:48:17,331 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d320e0f206c4773adc6b9aaa4bad1ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731397693227 2024-11-12T07:48:17,332 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 19da1e97d21d4f57bfae6566a7769fc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731397694385 2024-11-12T07:48:17,332 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e92d82ffcb6451cbbf3fb4dab1da527, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731397695548 2024-11-12T07:48:17,334 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/A of 299b057d313257fac3b10771c8ca0f20 into db51becab35043858815203259c2182f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:17,334 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:17,334 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/A, priority=13, startTime=1731397696879; duration=0sec 2024-11-12T07:48:17,334 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:17,334 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:A 2024-11-12T07:48:17,340 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#C#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:17,341 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/fdbc49e0b5444f269f3b32c99aa45a74 is 50, key is test_row_0/C:col10/1731397695548/Put/seqid=0 2024-11-12T07:48:17,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742012_1188 (size=12561) 2024-11-12T07:48:17,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397757428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:17,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:17,439 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c15cdbc1df4240e2be3055bfe02af5f7_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c15cdbc1df4240e2be3055bfe02af5f7_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/492637b156fb4c1ba512e0a19c4c2eb0, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:17,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/492637b156fb4c1ba512e0a19c4c2eb0 is 175, key is test_row_0/A:col10/1731397695985/Put/seqid=0 2024-11-12T07:48:17,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742013_1189 (size=31105) 2024-11-12T07:48:17,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397757738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:17,750 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/fdbc49e0b5444f269f3b32c99aa45a74 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/fdbc49e0b5444f269f3b32c99aa45a74 2024-11-12T07:48:17,755 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/C of 299b057d313257fac3b10771c8ca0f20 into fdbc49e0b5444f269f3b32c99aa45a74(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:17,755 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:17,755 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/C, priority=13, startTime=1731397696879; duration=0sec 2024-11-12T07:48:17,755 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:17,756 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:C 2024-11-12T07:48:17,857 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/492637b156fb4c1ba512e0a19c4c2eb0 2024-11-12T07:48:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4cfee43296624f1c965c455ea1fd69fb is 50, key is test_row_0/B:col10/1731397695985/Put/seqid=0 2024-11-12T07:48:17,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742014_1190 (size=12151) 2024-11-12T07:48:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T07:48:18,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397758148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:18,152 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:18,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:18,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:18,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397758158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:18,162 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:18,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397758248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:18,315 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4cfee43296624f1c965c455ea1fd69fb 2024-11-12T07:48:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/b8175f11cb71488ca0289d20466902d4 is 50, key is test_row_0/C:col10/1731397695985/Put/seqid=0 2024-11-12T07:48:18,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742015_1191 (size=12151) 2024-11-12T07:48:18,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397758597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:18,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397758598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:18,598 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_0' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,598 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:18,728 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/b8175f11cb71488ca0289d20466902d4 2024-11-12T07:48:18,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/492637b156fb4c1ba512e0a19c4c2eb0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/492637b156fb4c1ba512e0a19c4c2eb0 2024-11-12T07:48:18,737 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/492637b156fb4c1ba512e0a19c4c2eb0, entries=150, sequenceid=198, filesize=30.4 K 2024-11-12T07:48:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4cfee43296624f1c965c455ea1fd69fb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4cfee43296624f1c965c455ea1fd69fb 2024-11-12T07:48:18,742 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4cfee43296624f1c965c455ea1fd69fb, entries=150, sequenceid=198, filesize=11.9 K 2024-11-12T07:48:18,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/b8175f11cb71488ca0289d20466902d4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/b8175f11cb71488ca0289d20466902d4 2024-11-12T07:48:18,748 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/b8175f11cb71488ca0289d20466902d4, entries=150, sequenceid=198, filesize=11.9 K 2024-11-12T07:48:18,749 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 299b057d313257fac3b10771c8ca0f20 in 1726ms, sequenceid=198, compaction requested=false 2024-11-12T07:48:18,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:18,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:18,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-12T07:48:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-12T07:48:18,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-12T07:48:18,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8040 sec 2024-11-12T07:48:18,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.8140 sec 2024-11-12T07:48:19,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:19,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:48:19,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:19,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:19,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111242d43af324a749eea53e0e091c36140d_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397697088/Put/seqid=0 2024-11-12T07:48:19,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742016_1192 (size=12304) 2024-11-12T07:48:19,271 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:19,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111242d43af324a749eea53e0e091c36140d_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111242d43af324a749eea53e0e091c36140d_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:19,276 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/a3e2a9427fd740409e0449a98e8fa618, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:19,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/a3e2a9427fd740409e0449a98e8fa618 is 175, key is test_row_0/A:col10/1731397697088/Put/seqid=0 2024-11-12T07:48:19,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742017_1193 (size=31105) 2024-11-12T07:48:19,285 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/a3e2a9427fd740409e0449a98e8fa618 2024-11-12T07:48:19,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/71335ba6a4524cd0b8a6a0e206af3eb4 is 50, key is test_row_0/B:col10/1731397697088/Put/seqid=0 2024-11-12T07:48:19,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742018_1194 (size=12151) 2024-11-12T07:48:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397759331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397759438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:19,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397759648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:19,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:19,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/71335ba6a4524cd0b8a6a0e206af3eb4 2024-11-12T07:48:19,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/13eb738909744176b23cee768a46953d is 50, key is test_row_0/C:col10/1731397697088/Put/seqid=0 2024-11-12T07:48:19,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742019_1195 (size=12151) 2024-11-12T07:48:19,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/13eb738909744176b23cee768a46953d 2024-11-12T07:48:19,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/a3e2a9427fd740409e0449a98e8fa618 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/a3e2a9427fd740409e0449a98e8fa618 2024-11-12T07:48:19,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/a3e2a9427fd740409e0449a98e8fa618, entries=150, sequenceid=213, filesize=30.4 K 2024-11-12T07:48:19,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/71335ba6a4524cd0b8a6a0e206af3eb4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/71335ba6a4524cd0b8a6a0e206af3eb4 2024-11-12T07:48:19,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/71335ba6a4524cd0b8a6a0e206af3eb4, entries=150, sequenceid=213, filesize=11.9 K 2024-11-12T07:48:19,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/13eb738909744176b23cee768a46953d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/13eb738909744176b23cee768a46953d 2024-11-12T07:48:19,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/13eb738909744176b23cee768a46953d, entries=150, sequenceid=213, filesize=11.9 K 2024-11-12T07:48:19,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 299b057d313257fac3b10771c8ca0f20 in 482ms, sequenceid=213, compaction requested=true 2024-11-12T07:48:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:19,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:19,741 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:19,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:19,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:19,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:19,742 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:19,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:19,743 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:19,743 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:19,743 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/A is initiating minor compaction (all files) 2024-11-12T07:48:19,743 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/B is initiating minor compaction (all files) 2024-11-12T07:48:19,743 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/B in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:19,743 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/A in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:19,743 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce09371cb1464a3e9761e48ea8022c7b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4cfee43296624f1c965c455ea1fd69fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/71335ba6a4524cd0b8a6a0e206af3eb4] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=36.0 K 2024-11-12T07:48:19,743 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/db51becab35043858815203259c2182f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/492637b156fb4c1ba512e0a19c4c2eb0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/a3e2a9427fd740409e0449a98e8fa618] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=91.5 K 2024-11-12T07:48:19,743 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:19,743 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/db51becab35043858815203259c2182f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/492637b156fb4c1ba512e0a19c4c2eb0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/a3e2a9427fd740409e0449a98e8fa618] 2024-11-12T07:48:19,743 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ce09371cb1464a3e9761e48ea8022c7b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731397695548 2024-11-12T07:48:19,743 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting db51becab35043858815203259c2182f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731397695548 2024-11-12T07:48:19,744 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cfee43296624f1c965c455ea1fd69fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731397695966 2024-11-12T07:48:19,744 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492637b156fb4c1ba512e0a19c4c2eb0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731397695966 2024-11-12T07:48:19,744 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 71335ba6a4524cd0b8a6a0e206af3eb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731397697088 2024-11-12T07:48:19,744 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e2a9427fd740409e0449a98e8fa618, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731397697088 2024-11-12T07:48:19,751 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:19,751 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/0809f7fe133a438c819c9def10c3ab94 is 50, key is test_row_0/B:col10/1731397697088/Put/seqid=0 2024-11-12T07:48:19,753 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:19,756 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111267fc9df335f4497fa8e93e11a60cfd26_299b057d313257fac3b10771c8ca0f20 store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:19,759 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111267fc9df335f4497fa8e93e11a60cfd26_299b057d313257fac3b10771c8ca0f20, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:19,759 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111267fc9df335f4497fa8e93e11a60cfd26_299b057d313257fac3b10771c8ca0f20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:19,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742020_1196 (size=12663) 2024-11-12T07:48:19,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742021_1197 (size=4469) 2024-11-12T07:48:19,769 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/0809f7fe133a438c819c9def10c3ab94 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0809f7fe133a438c819c9def10c3ab94 2024-11-12T07:48:19,774 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/B of 299b057d313257fac3b10771c8ca0f20 into 0809f7fe133a438c819c9def10c3ab94(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:19,774 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:19,775 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/B, priority=13, startTime=1731397699741; duration=0sec 2024-11-12T07:48:19,775 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:19,775 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:B 2024-11-12T07:48:19,775 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:19,776 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:19,776 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/C is initiating minor compaction (all files) 2024-11-12T07:48:19,776 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/C in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:19,776 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/fdbc49e0b5444f269f3b32c99aa45a74, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/b8175f11cb71488ca0289d20466902d4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/13eb738909744176b23cee768a46953d] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=36.0 K 2024-11-12T07:48:19,777 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting fdbc49e0b5444f269f3b32c99aa45a74, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731397695548 2024-11-12T07:48:19,777 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting b8175f11cb71488ca0289d20466902d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731397695966 2024-11-12T07:48:19,777 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 13eb738909744176b23cee768a46953d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731397697088 2024-11-12T07:48:19,785 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:19,786 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/dcea1bddb6fe4883835c30b750daf916 is 50, key is test_row_0/C:col10/1731397697088/Put/seqid=0 2024-11-12T07:48:19,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742022_1198 (size=12663) 2024-11-12T07:48:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:19,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T07:48:19,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:19,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:19,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112da54d584b1d74ca885c1733cb328cc04_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397699330/Put/seqid=0 2024-11-12T07:48:19,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742023_1199 (size=12304) 2024-11-12T07:48:19,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:19,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397759989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:19,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:19,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:20,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397760098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:20,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T07:48:20,108 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:20,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-12T07:48:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T07:48:20,110 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:20,110 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:20,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:20,168 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:20,169 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/9a8efe32c830476a9f06a48126e0260f is 175, key is test_row_0/A:col10/1731397697088/Put/seqid=0 2024-11-12T07:48:20,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397760168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:20,169 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:20,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742024_1200 (size=31617) 2024-11-12T07:48:20,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:20,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397760178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:20,183 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:20,199 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/dcea1bddb6fe4883835c30b750daf916 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/dcea1bddb6fe4883835c30b750daf916 2024-11-12T07:48:20,204 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/C of 299b057d313257fac3b10771c8ca0f20 into dcea1bddb6fe4883835c30b750daf916(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:20,204 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:20,204 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/C, priority=13, startTime=1731397699742; duration=0sec 2024-11-12T07:48:20,204 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:20,204 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:C 2024-11-12T07:48:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T07:48:20,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397760307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:20,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:20,379 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112da54d584b1d74ca885c1733cb328cc04_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112da54d584b1d74ca885c1733cb328cc04_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:20,380 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/06d09ad341ed44bb89462e1c428c15aa, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:20,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/06d09ad341ed44bb89462e1c428c15aa is 175, key is test_row_0/A:col10/1731397699330/Put/seqid=0 2024-11-12T07:48:20,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742025_1201 (size=31105) 2024-11-12T07:48:20,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T07:48:20,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,578 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/9a8efe32c830476a9f06a48126e0260f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/9a8efe32c830476a9f06a48126e0260f 2024-11-12T07:48:20,583 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/A of 299b057d313257fac3b10771c8ca0f20 into 9a8efe32c830476a9f06a48126e0260f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:20,583 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:20,583 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/A, priority=13, startTime=1731397699741; duration=0sec 2024-11-12T07:48:20,583 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:20,583 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:A 2024-11-12T07:48:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397760618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:20,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:20,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T07:48:20,786 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/06d09ad341ed44bb89462e1c428c15aa 2024-11-12T07:48:20,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4f03c058b23b447dbc547f350152b8c6 is 50, key is test_row_0/B:col10/1731397699330/Put/seqid=0 2024-11-12T07:48:20,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742026_1202 (size=12151) 2024-11-12T07:48:20,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:20,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397761128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:21,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4f03c058b23b447dbc547f350152b8c6 2024-11-12T07:48:21,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/8b9d130f826c4958ac592d06a8ad7887 is 50, key is test_row_0/C:col10/1731397699330/Put/seqid=0 2024-11-12T07:48:21,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742027_1203 (size=12151) 2024-11-12T07:48:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T07:48:21,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:21,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:21,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:21,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:21,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:21,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/8b9d130f826c4958ac592d06a8ad7887 2024-11-12T07:48:21,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/06d09ad341ed44bb89462e1c428c15aa as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/06d09ad341ed44bb89462e1c428c15aa 2024-11-12T07:48:21,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/06d09ad341ed44bb89462e1c428c15aa, entries=150, sequenceid=235, filesize=30.4 K 2024-11-12T07:48:21,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/4f03c058b23b447dbc547f350152b8c6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4f03c058b23b447dbc547f350152b8c6 2024-11-12T07:48:21,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4f03c058b23b447dbc547f350152b8c6, entries=150, sequenceid=235, filesize=11.9 K 2024-11-12T07:48:21,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/8b9d130f826c4958ac592d06a8ad7887 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b9d130f826c4958ac592d06a8ad7887 2024-11-12T07:48:21,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b9d130f826c4958ac592d06a8ad7887, entries=150, sequenceid=235, filesize=11.9 K 2024-11-12T07:48:21,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 299b057d313257fac3b10771c8ca0f20 in 1678ms, sequenceid=235, compaction requested=false 2024-11-12T07:48:21,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:21,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-12T07:48:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:21,639 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T07:48:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:21,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123fa92756968949898abcdb8d4759be22_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397699973/Put/seqid=0 2024-11-12T07:48:21,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742028_1204 (size=12304) 2024-11-12T07:48:22,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:22,062 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123fa92756968949898abcdb8d4759be22_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123fa92756968949898abcdb8d4759be22_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/64629560ccb04419bf377f62ff8f2b35, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:22,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/64629560ccb04419bf377f62ff8f2b35 is 175, key is test_row_0/A:col10/1731397699973/Put/seqid=0 2024-11-12T07:48:22,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742029_1205 (size=31105) 2024-11-12T07:48:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:22,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:22,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397702161, max timestamp=1731397702199Num uniques 859; 2024-11-12T07:48:22,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397702161, max timestamp=1731397702199Num uniques 859; 2024-11-12T07:48:22,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397702161, max timestamp=1731397702199Num uniques 859; 2024-11-12T07:48:22,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:48:22,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:48:22,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:48:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T07:48:22,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397762262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:22,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397762378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:22,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:22,471 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/64629560ccb04419bf377f62ff8f2b35 2024-11-12T07:48:22,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/bb42c75ea2c741c4bbafd1385723568f is 50, key is test_row_0/B:col10/1731397699973/Put/seqid=0 2024-11-12T07:48:22,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742030_1206 (size=12151) 2024-11-12T07:48:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397762588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38846 deadline: 1731397762628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:22,629 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:22,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38874 deadline: 1731397762637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:22,639 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_0' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:22,882 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/bb42c75ea2c741c4bbafd1385723568f 2024-11-12T07:48:22,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/194d93f8480f493db69722dfb6730ea4 is 50, key is test_row_0/C:col10/1731397699973/Put/seqid=0 2024-11-12T07:48:22,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742031_1207 (size=12151) 2024-11-12T07:48:22,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397762897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:23,297 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/194d93f8480f493db69722dfb6730ea4 2024-11-12T07:48:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/64629560ccb04419bf377f62ff8f2b35 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64629560ccb04419bf377f62ff8f2b35 2024-11-12T07:48:23,353 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64629560ccb04419bf377f62ff8f2b35, entries=150, sequenceid=248, filesize=30.4 K 2024-11-12T07:48:23,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/bb42c75ea2c741c4bbafd1385723568f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bb42c75ea2c741c4bbafd1385723568f 2024-11-12T07:48:23,358 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bb42c75ea2c741c4bbafd1385723568f, entries=150, sequenceid=248, filesize=11.9 K 2024-11-12T07:48:23,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/194d93f8480f493db69722dfb6730ea4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/194d93f8480f493db69722dfb6730ea4 2024-11-12T07:48:23,363 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/194d93f8480f493db69722dfb6730ea4, entries=150, sequenceid=248, filesize=11.9 K 2024-11-12T07:48:23,364 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 299b057d313257fac3b10771c8ca0f20 in 1725ms, sequenceid=248, compaction requested=true 2024-11-12T07:48:23,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:23,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:23,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-12T07:48:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-12T07:48:23,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T07:48:23,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2550 sec 2024-11-12T07:48:23,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.2600 sec 2024-11-12T07:48:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:23,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T07:48:23,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:23,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:23,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:23,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:23,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:23,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:23,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e47b1f04ffa5401b9c4ba2ad4fb62e4a_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397703409/Put/seqid=0 2024-11-12T07:48:23,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742032_1208 (size=14994) 2024-11-12T07:48:23,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:23,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e47b1f04ffa5401b9c4ba2ad4fb62e4a_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e47b1f04ffa5401b9c4ba2ad4fb62e4a_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:23,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/31357c3bf0794f68b4b39567c8524e7f, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:23,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/31357c3bf0794f68b4b39567c8524e7f is 175, key is test_row_0/A:col10/1731397703409/Put/seqid=0 2024-11-12T07:48:23,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742033_1209 (size=39949) 2024-11-12T07:48:23,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397763446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:23,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:23,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:23,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:23,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397763557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397763768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:23,834 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/31357c3bf0794f68b4b39567c8524e7f 2024-11-12T07:48:23,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/bd80382482324994a5bd63b22066f6da is 50, key is test_row_0/B:col10/1731397703409/Put/seqid=0 2024-11-12T07:48:23,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742034_1210 (size=12301) 2024-11-12T07:48:24,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397764077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38836 deadline: 1731397764208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:24,210 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38812 deadline: 1731397764218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:24,223 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 299b057d313257fac3b10771c8ca0f20 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:24,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/bd80382482324994a5bd63b22066f6da 2024-11-12T07:48:24,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d3f1a2de914346da8b13efbe2c2479ef is 50, key is test_row_0/C:col10/1731397703409/Put/seqid=0 2024-11-12T07:48:24,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742035_1211 (size=12301) 2024-11-12T07:48:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-12T07:48:24,278 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:24,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-12T07:48:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T07:48:24,280 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:24,280 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:24,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T07:48:24,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T07:48:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:24,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:24,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:24,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397764588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:24,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:24,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T07:48:24,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:24,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:24,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T07:48:24,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d3f1a2de914346da8b13efbe2c2479ef 2024-11-12T07:48:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/31357c3bf0794f68b4b39567c8524e7f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/31357c3bf0794f68b4b39567c8524e7f 2024-11-12T07:48:24,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/31357c3bf0794f68b4b39567c8524e7f, entries=200, sequenceid=284, filesize=39.0 K 2024-11-12T07:48:24,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/bd80382482324994a5bd63b22066f6da as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bd80382482324994a5bd63b22066f6da 2024-11-12T07:48:24,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bd80382482324994a5bd63b22066f6da, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T07:48:24,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d3f1a2de914346da8b13efbe2c2479ef as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d3f1a2de914346da8b13efbe2c2479ef 2024-11-12T07:48:24,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d3f1a2de914346da8b13efbe2c2479ef, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T07:48:24,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 299b057d313257fac3b10771c8ca0f20 in 1272ms, sequenceid=284, compaction requested=true 2024-11-12T07:48:24,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:24,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:24,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:24,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:24,682 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:48:24,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:24,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:24,682 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:48:24,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:24,683 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133776 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:48:24,683 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:48:24,683 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/A is initiating minor compaction (all files) 2024-11-12T07:48:24,683 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/B is initiating minor compaction (all files) 2024-11-12T07:48:24,683 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/A in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:24,683 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/B in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:24,683 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/9a8efe32c830476a9f06a48126e0260f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/06d09ad341ed44bb89462e1c428c15aa, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64629560ccb04419bf377f62ff8f2b35, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/31357c3bf0794f68b4b39567c8524e7f] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=130.6 K 2024-11-12T07:48:24,683 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0809f7fe133a438c819c9def10c3ab94, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4f03c058b23b447dbc547f350152b8c6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bb42c75ea2c741c4bbafd1385723568f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bd80382482324994a5bd63b22066f6da] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=48.1 K 2024-11-12T07:48:24,683 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:24,684 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/9a8efe32c830476a9f06a48126e0260f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/06d09ad341ed44bb89462e1c428c15aa, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64629560ccb04419bf377f62ff8f2b35, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/31357c3bf0794f68b4b39567c8524e7f] 2024-11-12T07:48:24,684 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 0809f7fe133a438c819c9def10c3ab94, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731397697088 2024-11-12T07:48:24,684 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a8efe32c830476a9f06a48126e0260f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731397697088 2024-11-12T07:48:24,684 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f03c058b23b447dbc547f350152b8c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731397699323 2024-11-12T07:48:24,684 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d09ad341ed44bb89462e1c428c15aa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731397699323 2024-11-12T07:48:24,684 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting bb42c75ea2c741c4bbafd1385723568f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731397699973 2024-11-12T07:48:24,684 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64629560ccb04419bf377f62ff8f2b35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731397699973 2024-11-12T07:48:24,685 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting bd80382482324994a5bd63b22066f6da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731397702258 2024-11-12T07:48:24,685 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31357c3bf0794f68b4b39567c8524e7f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731397702245 2024-11-12T07:48:24,693 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:24,694 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:24,695 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411128ab61b60dc8c4ade91050e1a31199e61_299b057d313257fac3b10771c8ca0f20 store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:24,695 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/5625bbcc2210474c99cc4545df7c29f5 is 50, key is test_row_0/B:col10/1731397703409/Put/seqid=0 2024-11-12T07:48:24,698 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411128ab61b60dc8c4ade91050e1a31199e61_299b057d313257fac3b10771c8ca0f20, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:24,698 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128ab61b60dc8c4ade91050e1a31199e61_299b057d313257fac3b10771c8ca0f20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:24,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742036_1212 (size=12949) 2024-11-12T07:48:24,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742037_1213 (size=4469) 2024-11-12T07:48:24,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T07:48:24,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:24,747 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:48:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:24,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121ab92b05d9334b17b64a551cdc5ce7d4_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397703437/Put/seqid=0 2024-11-12T07:48:24,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742038_1214 (size=12454) 2024-11-12T07:48:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T07:48:25,107 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/5625bbcc2210474c99cc4545df7c29f5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5625bbcc2210474c99cc4545df7c29f5 2024-11-12T07:48:25,112 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/B of 299b057d313257fac3b10771c8ca0f20 into 5625bbcc2210474c99cc4545df7c29f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:25,112 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:25,112 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/B, priority=12, startTime=1731397704682; duration=0sec 2024-11-12T07:48:25,112 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:25,112 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:B 2024-11-12T07:48:25,112 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:48:25,113 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:48:25,114 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/C is initiating minor compaction (all files) 2024-11-12T07:48:25,114 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/C in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:25,114 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/dcea1bddb6fe4883835c30b750daf916, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b9d130f826c4958ac592d06a8ad7887, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/194d93f8480f493db69722dfb6730ea4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d3f1a2de914346da8b13efbe2c2479ef] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=48.1 K 2024-11-12T07:48:25,114 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting dcea1bddb6fe4883835c30b750daf916, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731397697088 2024-11-12T07:48:25,114 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b9d130f826c4958ac592d06a8ad7887, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731397699323 2024-11-12T07:48:25,115 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 194d93f8480f493db69722dfb6730ea4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731397699973 2024-11-12T07:48:25,117 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d3f1a2de914346da8b13efbe2c2479ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731397702258 2024-11-12T07:48:25,117 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:25,118 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/64ece1e3516b4a5882b1a3843b90cae2 is 175, key is test_row_0/A:col10/1731397703409/Put/seqid=0 2024-11-12T07:48:25,128 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:25,128 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d6e145ddba04404b94dcabefcf13f802 is 50, key is test_row_0/C:col10/1731397703409/Put/seqid=0 2024-11-12T07:48:25,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742039_1215 (size=31903) 2024-11-12T07:48:25,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742040_1216 (size=12949) 2024-11-12T07:48:25,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:25,165 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121ab92b05d9334b17b64a551cdc5ce7d4_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121ab92b05d9334b17b64a551cdc5ce7d4_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:25,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/f64ba0edbbbc4970ae7fd7978e13edff, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/f64ba0edbbbc4970ae7fd7978e13edff is 175, key is test_row_0/A:col10/1731397703437/Put/seqid=0 2024-11-12T07:48:25,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742041_1217 (size=31255) 2024-11-12T07:48:25,172 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/f64ba0edbbbc4970ae7fd7978e13edff 2024-11-12T07:48:25,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/c1b317e8ecf846adbeba79b9b7574cc5 is 50, key is test_row_0/B:col10/1731397703437/Put/seqid=0 2024-11-12T07:48:25,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742042_1218 (size=12301) 2024-11-12T07:48:25,192 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/c1b317e8ecf846adbeba79b9b7574cc5 2024-11-12T07:48:25,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/076a86cdf97141d992a991a42aea8309 is 50, key is test_row_0/C:col10/1731397703437/Put/seqid=0 2024-11-12T07:48:25,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742043_1219 (size=12301) 2024-11-12T07:48:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T07:48:25,534 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/64ece1e3516b4a5882b1a3843b90cae2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64ece1e3516b4a5882b1a3843b90cae2 2024-11-12T07:48:25,536 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/d6e145ddba04404b94dcabefcf13f802 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d6e145ddba04404b94dcabefcf13f802 2024-11-12T07:48:25,539 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/A of 299b057d313257fac3b10771c8ca0f20 into 64ece1e3516b4a5882b1a3843b90cae2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:25,539 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:25,539 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/A, priority=12, startTime=1731397704682; duration=0sec 2024-11-12T07:48:25,539 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:25,539 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:A 2024-11-12T07:48:25,542 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/C of 299b057d313257fac3b10771c8ca0f20 into d6e145ddba04404b94dcabefcf13f802(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:25,542 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:25,542 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/C, priority=12, startTime=1731397704682; duration=0sec 2024-11-12T07:48:25,542 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:25,542 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:C 2024-11-12T07:48:25,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. as already flushing 2024-11-12T07:48:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:25,606 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/076a86cdf97141d992a991a42aea8309 2024-11-12T07:48:25,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/f64ba0edbbbc4970ae7fd7978e13edff as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f64ba0edbbbc4970ae7fd7978e13edff 2024-11-12T07:48:25,615 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f64ba0edbbbc4970ae7fd7978e13edff, entries=150, sequenceid=294, filesize=30.5 K 2024-11-12T07:48:25,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/c1b317e8ecf846adbeba79b9b7574cc5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/c1b317e8ecf846adbeba79b9b7574cc5 2024-11-12T07:48:25,623 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/c1b317e8ecf846adbeba79b9b7574cc5, entries=150, sequenceid=294, filesize=12.0 K 2024-11-12T07:48:25,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/076a86cdf97141d992a991a42aea8309 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/076a86cdf97141d992a991a42aea8309 2024-11-12T07:48:25,629 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/076a86cdf97141d992a991a42aea8309, entries=150, sequenceid=294, filesize=12.0 K 2024-11-12T07:48:25,630 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=73.80 KB/75570 for 299b057d313257fac3b10771c8ca0f20 in 883ms, sequenceid=294, compaction requested=false 2024-11-12T07:48:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:25,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-12T07:48:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-12T07:48:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:25,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T07:48:25,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:25,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:25,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:25,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:25,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:25,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:25,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-12T07:48:25,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3510 sec 2024-11-12T07:48:25,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.3560 sec 2024-11-12T07:48:25,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111293b5f32ce50a43e48524ad34bce45ba3_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397705627/Put/seqid=0 2024-11-12T07:48:25,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742044_1220 (size=17534) 2024-11-12T07:48:25,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397765687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:25,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397765798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397766008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:26,045 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:26,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111293b5f32ce50a43e48524ad34bce45ba3_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293b5f32ce50a43e48524ad34bce45ba3_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:26,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/c45932916b1d4abd9226883d09d0ff8e, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:26,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/c45932916b1d4abd9226883d09d0ff8e is 175, key is test_row_0/A:col10/1731397705627/Put/seqid=0 2024-11-12T07:48:26,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742045_1221 (size=48639) 2024-11-12T07:48:26,311 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-12T07:48:26,311 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:26,311 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:26,311 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:26,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:26,315 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-12T07:48:26,315 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:26,315 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:26,315 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:26,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:26,315 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-12T07:48:26,315 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:26,315 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:26,316 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:26,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397766318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:26,320 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-12T07:48:26,320 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:26,320 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:26,320 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:26,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T07:48:26,428 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:26,453 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/c45932916b1d4abd9226883d09d0ff8e 2024-11-12T07:48:26,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/22860a5027ea49feb976ece489450cec is 50, key is test_row_0/B:col10/1731397705627/Put/seqid=0 2024-11-12T07:48:26,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742046_1222 (size=12301) 2024-11-12T07:48:26,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38864 deadline: 1731397766828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:26,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:48:26,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/22860a5027ea49feb976ece489450cec 2024-11-12T07:48:26,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/10626efe63b04535a52d390cb7cf0833 is 50, key is test_row_0/C:col10/1731397705627/Put/seqid=0 2024-11-12T07:48:26,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742047_1223 (size=12301) 2024-11-12T07:48:27,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/10626efe63b04535a52d390cb7cf0833 2024-11-12T07:48:27,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/c45932916b1d4abd9226883d09d0ff8e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c45932916b1d4abd9226883d09d0ff8e 2024-11-12T07:48:27,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c45932916b1d4abd9226883d09d0ff8e, entries=250, sequenceid=312, filesize=47.5 K 2024-11-12T07:48:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/22860a5027ea49feb976ece489450cec as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/22860a5027ea49feb976ece489450cec 2024-11-12T07:48:27,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/22860a5027ea49feb976ece489450cec, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T07:48:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/10626efe63b04535a52d390cb7cf0833 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/10626efe63b04535a52d390cb7cf0833 2024-11-12T07:48:27,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/10626efe63b04535a52d390cb7cf0833, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T07:48:27,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 299b057d313257fac3b10771c8ca0f20 in 1676ms, sequenceid=312, compaction requested=true 2024-11-12T07:48:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:27,307 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 299b057d313257fac3b10771c8ca0f20:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:27,307 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:27,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:27,308 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:27,308 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:27,308 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/A is initiating minor compaction (all files) 2024-11-12T07:48:27,308 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/B is initiating minor compaction (all files) 2024-11-12T07:48:27,308 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/A in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:27,309 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/B in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:27,309 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64ece1e3516b4a5882b1a3843b90cae2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f64ba0edbbbc4970ae7fd7978e13edff, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c45932916b1d4abd9226883d09d0ff8e] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=109.2 K 2024-11-12T07:48:27,309 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5625bbcc2210474c99cc4545df7c29f5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/c1b317e8ecf846adbeba79b9b7574cc5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/22860a5027ea49feb976ece489450cec] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=36.7 K 2024-11-12T07:48:27,309 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:27,309 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64ece1e3516b4a5882b1a3843b90cae2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f64ba0edbbbc4970ae7fd7978e13edff, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c45932916b1d4abd9226883d09d0ff8e] 2024-11-12T07:48:27,309 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5625bbcc2210474c99cc4545df7c29f5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731397702258 2024-11-12T07:48:27,309 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ece1e3516b4a5882b1a3843b90cae2, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731397702258 2024-11-12T07:48:27,309 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b317e8ecf846adbeba79b9b7574cc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731397703429 2024-11-12T07:48:27,309 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f64ba0edbbbc4970ae7fd7978e13edff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731397703429 2024-11-12T07:48:27,310 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 22860a5027ea49feb976ece489450cec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731397705625 2024-11-12T07:48:27,310 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45932916b1d4abd9226883d09d0ff8e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731397705604 2024-11-12T07:48:27,320 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#B#compaction#186 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:27,321 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/730b46c9a59148a192cdf36d0233b027 is 50, key is test_row_0/B:col10/1731397705627/Put/seqid=0 2024-11-12T07:48:27,321 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:27,324 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112de4c640cd1ce4db5aacd1d68d951be1d_299b057d313257fac3b10771c8ca0f20 store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:27,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742048_1224 (size=13051) 2024-11-12T07:48:27,355 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112de4c640cd1ce4db5aacd1d68d951be1d_299b057d313257fac3b10771c8ca0f20, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:27,355 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112de4c640cd1ce4db5aacd1d68d951be1d_299b057d313257fac3b10771c8ca0f20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:27,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742049_1225 (size=4469) 2024-11-12T07:48:27,734 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/730b46c9a59148a192cdf36d0233b027 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/730b46c9a59148a192cdf36d0233b027 2024-11-12T07:48:27,740 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/B of 299b057d313257fac3b10771c8ca0f20 into 730b46c9a59148a192cdf36d0233b027(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:27,740 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:27,740 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/B, priority=13, startTime=1731397707307; duration=0sec 2024-11-12T07:48:27,740 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:27,740 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:B 2024-11-12T07:48:27,740 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:27,741 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:27,741 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 299b057d313257fac3b10771c8ca0f20/C is initiating minor compaction (all files) 2024-11-12T07:48:27,741 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 299b057d313257fac3b10771c8ca0f20/C in TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:27,742 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d6e145ddba04404b94dcabefcf13f802, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/076a86cdf97141d992a991a42aea8309, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/10626efe63b04535a52d390cb7cf0833] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp, totalSize=36.7 K 2024-11-12T07:48:27,742 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e145ddba04404b94dcabefcf13f802, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731397702258 2024-11-12T07:48:27,742 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 076a86cdf97141d992a991a42aea8309, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731397703429 2024-11-12T07:48:27,743 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 10626efe63b04535a52d390cb7cf0833, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731397705625 2024-11-12T07:48:27,754 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#C#compaction#188 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:27,754 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/8b7edf2e638c40cba0da0e117b2823a4 is 50, key is test_row_0/C:col10/1731397705627/Put/seqid=0 2024-11-12T07:48:27,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742050_1226 (size=13051) 2024-11-12T07:48:27,760 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 299b057d313257fac3b10771c8ca0f20#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:27,761 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/18df1ef01f614c09a2ddf447a659ea5c is 175, key is test_row_0/A:col10/1731397705627/Put/seqid=0 2024-11-12T07:48:27,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742051_1227 (size=32005) 2024-11-12T07:48:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:27,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:48:27,850 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-11-12T07:48:27,850 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:27,850 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:27,850 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:27,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:27,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:27,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112987368e281ad4ba1955904601e68a517_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397705681/Put/seqid=0 2024-11-12T07:48:27,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742052_1228 (size=12454) 2024-11-12T07:48:28,166 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/8b7edf2e638c40cba0da0e117b2823a4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b7edf2e638c40cba0da0e117b2823a4 2024-11-12T07:48:28,171 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/18df1ef01f614c09a2ddf447a659ea5c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/18df1ef01f614c09a2ddf447a659ea5c 2024-11-12T07:48:28,172 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/C of 299b057d313257fac3b10771c8ca0f20 into 8b7edf2e638c40cba0da0e117b2823a4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:28,172 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:28,172 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/C, priority=13, startTime=1731397707307; duration=0sec 2024-11-12T07:48:28,172 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:28,172 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:C 2024-11-12T07:48:28,176 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 299b057d313257fac3b10771c8ca0f20/A of 299b057d313257fac3b10771c8ca0f20 into 18df1ef01f614c09a2ddf447a659ea5c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:28,176 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:28,176 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., storeName=299b057d313257fac3b10771c8ca0f20/A, priority=13, startTime=1731397707307; duration=0sec 2024-11-12T07:48:28,176 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:28,176 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 299b057d313257fac3b10771c8ca0f20:A 2024-11-12T07:48:28,262 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:28,266 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112987368e281ad4ba1955904601e68a517_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112987368e281ad4ba1955904601e68a517_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:28,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/722480ac826844b282ceb7b8e4fa14d0, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:28,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/722480ac826844b282ceb7b8e4fa14d0 is 175, key is test_row_0/A:col10/1731397705681/Put/seqid=0 2024-11-12T07:48:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742053_1229 (size=31255) 2024-11-12T07:48:28,672 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/722480ac826844b282ceb7b8e4fa14d0 2024-11-12T07:48:28,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/47ceaf92dff94ff896b0d4b71330f875 is 50, key is test_row_0/B:col10/1731397705681/Put/seqid=0 2024-11-12T07:48:28,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742054_1230 (size=12301) 2024-11-12T07:48:29,129 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:48:29,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/47ceaf92dff94ff896b0d4b71330f875 2024-11-12T07:48:29,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/07a7446455cd4eeca003ca5da29125ef is 50, key is test_row_0/C:col10/1731397705681/Put/seqid=0 2024-11-12T07:48:29,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742055_1231 (size=12301) 2024-11-12T07:48:29,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/07a7446455cd4eeca003ca5da29125ef 2024-11-12T07:48:29,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/722480ac826844b282ceb7b8e4fa14d0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/722480ac826844b282ceb7b8e4fa14d0 2024-11-12T07:48:29,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/722480ac826844b282ceb7b8e4fa14d0, entries=150, sequenceid=331, filesize=30.5 K 2024-11-12T07:48:29,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/47ceaf92dff94ff896b0d4b71330f875 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/47ceaf92dff94ff896b0d4b71330f875 2024-11-12T07:48:29,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/47ceaf92dff94ff896b0d4b71330f875, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T07:48:29,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/07a7446455cd4eeca003ca5da29125ef as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/07a7446455cd4eeca003ca5da29125ef 2024-11-12T07:48:29,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/07a7446455cd4eeca003ca5da29125ef, entries=150, sequenceid=331, filesize=12.0 K 2024-11-12T07:48:29,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 299b057d313257fac3b10771c8ca0f20 in 1740ms, sequenceid=331, compaction requested=false 2024-11-12T07:48:29,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 299b057d313257fac3b10771c8ca0f20: 2024-11-12T07:48:32,691 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-12T07:48:32,691 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:32,691 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:32,692 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:32,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:32,732 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-12T07:48:32,732 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:32,732 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:32,732 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:32,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:34,265 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-12T07:48:34,265 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:34,265 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:34,266 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:34,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:34,311 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-12T07:48:34,312 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:34,312 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:34,312 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:34,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:34,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T07:48:34,312 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3701 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3380 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1440 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4317 rows 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1433 2024-11-12T07:48:34,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4298 rows 2024-11-12T07:48:34,313 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:48:34,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:48:34,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:34,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:34,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:34,316 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T07:48:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T07:48:34,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397714321"}]},"ts":"1731397714321"} 2024-11-12T07:48:34,323 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T07:48:34,323 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T07:48:34,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:48:34,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, UNASSIGN}] 2024-11-12T07:48:34,325 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, UNASSIGN 2024-11-12T07:48:34,326 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=299b057d313257fac3b10771c8ca0f20, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:34,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, UNASSIGN because future has completed 2024-11-12T07:48:34,328 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:48:34,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:48:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T07:48:34,480 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:34,480 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:48:34,480 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 299b057d313257fac3b10771c8ca0f20, disabling compactions & flushes 2024-11-12T07:48:34,480 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:34,480 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:34,480 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. after waiting 0 ms 2024-11-12T07:48:34,481 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:34,481 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 299b057d313257fac3b10771c8ca0f20 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T07:48:34,481 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=A 2024-11-12T07:48:34,481 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:34,481 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=B 2024-11-12T07:48:34,481 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:34,481 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 299b057d313257fac3b10771c8ca0f20, store=C 2024-11-12T07:48:34,481 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:34,491 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ae668a17aead422499c4c99a0dac3f28_299b057d313257fac3b10771c8ca0f20 is 50, key is test_row_0/A:col10/1731397712729/Put/seqid=0 2024-11-12T07:48:34,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742056_1232 (size=12454) 2024-11-12T07:48:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T07:48:34,897 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:34,908 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ae668a17aead422499c4c99a0dac3f28_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ae668a17aead422499c4c99a0dac3f28_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:34,909 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/2bb769d2299e484ebd44b31b1a1bc5e4, store: [table=TestAcidGuarantees family=A region=299b057d313257fac3b10771c8ca0f20] 2024-11-12T07:48:34,910 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/2bb769d2299e484ebd44b31b1a1bc5e4 is 175, key is test_row_0/A:col10/1731397712729/Put/seqid=0 2024-11-12T07:48:34,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742057_1233 (size=31255) 2024-11-12T07:48:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T07:48:35,315 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/2bb769d2299e484ebd44b31b1a1bc5e4 2024-11-12T07:48:35,327 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/0eeb7c50a73f48899a54a491e8abc945 is 50, key is test_row_0/B:col10/1731397712729/Put/seqid=0 2024-11-12T07:48:35,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742058_1234 (size=12301) 2024-11-12T07:48:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T07:48:35,733 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/0eeb7c50a73f48899a54a491e8abc945 2024-11-12T07:48:35,747 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/4695ce505c7c484daaa6a0ef69c0ebd4 is 50, key is test_row_0/C:col10/1731397712729/Put/seqid=0 2024-11-12T07:48:35,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742059_1235 (size=12301) 2024-11-12T07:48:36,153 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/4695ce505c7c484daaa6a0ef69c0ebd4 2024-11-12T07:48:36,163 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/A/2bb769d2299e484ebd44b31b1a1bc5e4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/2bb769d2299e484ebd44b31b1a1bc5e4 2024-11-12T07:48:36,168 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/2bb769d2299e484ebd44b31b1a1bc5e4, entries=150, sequenceid=340, filesize=30.5 K 2024-11-12T07:48:36,170 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/B/0eeb7c50a73f48899a54a491e8abc945 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0eeb7c50a73f48899a54a491e8abc945 2024-11-12T07:48:36,175 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0eeb7c50a73f48899a54a491e8abc945, entries=150, sequenceid=340, filesize=12.0 K 2024-11-12T07:48:36,176 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/.tmp/C/4695ce505c7c484daaa6a0ef69c0ebd4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4695ce505c7c484daaa6a0ef69c0ebd4 2024-11-12T07:48:36,180 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4695ce505c7c484daaa6a0ef69c0ebd4, entries=150, sequenceid=340, filesize=12.0 K 2024-11-12T07:48:36,181 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 299b057d313257fac3b10771c8ca0f20 in 1700ms, sequenceid=340, compaction requested=true 2024-11-12T07:48:36,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d06569a02fd949609e62d5e23c6309de, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/7215861668b74c09a525513166a85c08, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c44c05857f054f4fad618ab83b8a3bb9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/890cab92fd5641dd8b33ee20711c99f1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8f2db8af5a464fd58c45d030604fb70b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/ef064779c962485abeb67c3aebb71913, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/e9395082b741470789163eaea73ff65a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8d6ebd4e1be74db59d7c7cafb596b9fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f2828ea19dd447dfbcac36f4a39e69a7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d00ca1391dd34c5f8c30a1a784ec7132, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/3129c4d83bf64248bf75370701437367, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d84c7ce189084ef5935ee6dc4473ef61, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/db51becab35043858815203259c2182f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/492637b156fb4c1ba512e0a19c4c2eb0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/9a8efe32c830476a9f06a48126e0260f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/a3e2a9427fd740409e0449a98e8fa618, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/06d09ad341ed44bb89462e1c428c15aa, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64629560ccb04419bf377f62ff8f2b35, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/31357c3bf0794f68b4b39567c8524e7f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64ece1e3516b4a5882b1a3843b90cae2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f64ba0edbbbc4970ae7fd7978e13edff, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c45932916b1d4abd9226883d09d0ff8e] to archive 2024-11-12T07:48:36,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:48:36,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d06569a02fd949609e62d5e23c6309de to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d06569a02fd949609e62d5e23c6309de 2024-11-12T07:48:36,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/7215861668b74c09a525513166a85c08 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/7215861668b74c09a525513166a85c08 2024-11-12T07:48:36,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c44c05857f054f4fad618ab83b8a3bb9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c44c05857f054f4fad618ab83b8a3bb9 2024-11-12T07:48:36,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/890cab92fd5641dd8b33ee20711c99f1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/890cab92fd5641dd8b33ee20711c99f1 2024-11-12T07:48:36,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8f2db8af5a464fd58c45d030604fb70b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8f2db8af5a464fd58c45d030604fb70b 2024-11-12T07:48:36,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/ef064779c962485abeb67c3aebb71913 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/ef064779c962485abeb67c3aebb71913 2024-11-12T07:48:36,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/e9395082b741470789163eaea73ff65a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/e9395082b741470789163eaea73ff65a 2024-11-12T07:48:36,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8d6ebd4e1be74db59d7c7cafb596b9fb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/8d6ebd4e1be74db59d7c7cafb596b9fb 2024-11-12T07:48:36,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f2828ea19dd447dfbcac36f4a39e69a7 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f2828ea19dd447dfbcac36f4a39e69a7 2024-11-12T07:48:36,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d00ca1391dd34c5f8c30a1a784ec7132 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d00ca1391dd34c5f8c30a1a784ec7132 2024-11-12T07:48:36,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/3129c4d83bf64248bf75370701437367 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/3129c4d83bf64248bf75370701437367 2024-11-12T07:48:36,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d84c7ce189084ef5935ee6dc4473ef61 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/d84c7ce189084ef5935ee6dc4473ef61 2024-11-12T07:48:36,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/db51becab35043858815203259c2182f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/db51becab35043858815203259c2182f 2024-11-12T07:48:36,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/492637b156fb4c1ba512e0a19c4c2eb0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/492637b156fb4c1ba512e0a19c4c2eb0 2024-11-12T07:48:36,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/9a8efe32c830476a9f06a48126e0260f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/9a8efe32c830476a9f06a48126e0260f 2024-11-12T07:48:36,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/a3e2a9427fd740409e0449a98e8fa618 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/a3e2a9427fd740409e0449a98e8fa618 2024-11-12T07:48:36,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/06d09ad341ed44bb89462e1c428c15aa to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/06d09ad341ed44bb89462e1c428c15aa 2024-11-12T07:48:36,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64629560ccb04419bf377f62ff8f2b35 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64629560ccb04419bf377f62ff8f2b35 2024-11-12T07:48:36,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/31357c3bf0794f68b4b39567c8524e7f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/31357c3bf0794f68b4b39567c8524e7f 2024-11-12T07:48:36,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64ece1e3516b4a5882b1a3843b90cae2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/64ece1e3516b4a5882b1a3843b90cae2 2024-11-12T07:48:36,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f64ba0edbbbc4970ae7fd7978e13edff to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/f64ba0edbbbc4970ae7fd7978e13edff 2024-11-12T07:48:36,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c45932916b1d4abd9226883d09d0ff8e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/c45932916b1d4abd9226883d09d0ff8e 2024-11-12T07:48:36,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4531087f146e473691874f4ec27314fd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba1a4211157044fab2158a789b681d1c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba630b74d9d946c691d1c6ba875a776b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/8818d91e5ebd48e785dbd3a27220df10, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/a37260779d894db2901ad2f79e4ee32a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/69b311d273e745b68e536b328dcfa1eb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5fcd08576c47443fa05e13e98a53d26f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/27746e1378a142b0b04dd9a1bb723b27, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/33f1012808db46879cc9ce5aaf5c1cc7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/3d19013a89a44c058c920e14c828d19a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/e777a1fcf0c643c389edca032bc14fd1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce09371cb1464a3e9761e48ea8022c7b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce552e4071f64c0c9d3328ed265c011e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4cfee43296624f1c965c455ea1fd69fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0809f7fe133a438c819c9def10c3ab94, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/71335ba6a4524cd0b8a6a0e206af3eb4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4f03c058b23b447dbc547f350152b8c6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bb42c75ea2c741c4bbafd1385723568f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5625bbcc2210474c99cc4545df7c29f5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bd80382482324994a5bd63b22066f6da, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/c1b317e8ecf846adbeba79b9b7574cc5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/22860a5027ea49feb976ece489450cec] to archive 2024-11-12T07:48:36,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:48:36,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4531087f146e473691874f4ec27314fd to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4531087f146e473691874f4ec27314fd 2024-11-12T07:48:36,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba1a4211157044fab2158a789b681d1c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba1a4211157044fab2158a789b681d1c 2024-11-12T07:48:36,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba630b74d9d946c691d1c6ba875a776b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ba630b74d9d946c691d1c6ba875a776b 2024-11-12T07:48:36,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/8818d91e5ebd48e785dbd3a27220df10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/8818d91e5ebd48e785dbd3a27220df10 2024-11-12T07:48:36,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/a37260779d894db2901ad2f79e4ee32a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/a37260779d894db2901ad2f79e4ee32a 2024-11-12T07:48:36,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/69b311d273e745b68e536b328dcfa1eb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/69b311d273e745b68e536b328dcfa1eb 2024-11-12T07:48:36,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5fcd08576c47443fa05e13e98a53d26f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5fcd08576c47443fa05e13e98a53d26f 2024-11-12T07:48:36,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/27746e1378a142b0b04dd9a1bb723b27 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/27746e1378a142b0b04dd9a1bb723b27 2024-11-12T07:48:36,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/33f1012808db46879cc9ce5aaf5c1cc7 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/33f1012808db46879cc9ce5aaf5c1cc7 2024-11-12T07:48:36,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/3d19013a89a44c058c920e14c828d19a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/3d19013a89a44c058c920e14c828d19a 2024-11-12T07:48:36,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/e777a1fcf0c643c389edca032bc14fd1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/e777a1fcf0c643c389edca032bc14fd1 2024-11-12T07:48:36,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce09371cb1464a3e9761e48ea8022c7b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce09371cb1464a3e9761e48ea8022c7b 2024-11-12T07:48:36,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce552e4071f64c0c9d3328ed265c011e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/ce552e4071f64c0c9d3328ed265c011e 2024-11-12T07:48:36,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4cfee43296624f1c965c455ea1fd69fb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4cfee43296624f1c965c455ea1fd69fb 2024-11-12T07:48:36,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0809f7fe133a438c819c9def10c3ab94 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0809f7fe133a438c819c9def10c3ab94 2024-11-12T07:48:36,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/71335ba6a4524cd0b8a6a0e206af3eb4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/71335ba6a4524cd0b8a6a0e206af3eb4 2024-11-12T07:48:36,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4f03c058b23b447dbc547f350152b8c6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/4f03c058b23b447dbc547f350152b8c6 2024-11-12T07:48:36,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bb42c75ea2c741c4bbafd1385723568f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bb42c75ea2c741c4bbafd1385723568f 2024-11-12T07:48:36,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5625bbcc2210474c99cc4545df7c29f5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/5625bbcc2210474c99cc4545df7c29f5 2024-11-12T07:48:36,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bd80382482324994a5bd63b22066f6da to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/bd80382482324994a5bd63b22066f6da 2024-11-12T07:48:36,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/c1b317e8ecf846adbeba79b9b7574cc5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/c1b317e8ecf846adbeba79b9b7574cc5 2024-11-12T07:48:36,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/22860a5027ea49feb976ece489450cec to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/22860a5027ea49feb976ece489450cec 2024-11-12T07:48:36,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d5d456b5595a44d89e04b3c94801190c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/9196e3a1e6284bf4b7f2b5b911b39fc2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/35608baa4ca449eabb7c4f8768ebf11a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/778268ae7a3246d2bcc63d93f7baa470, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/c3747c43ebc14cf09ac8878360bef6f5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d7a290f01b0241d78f0cd8d1f6348796, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/e070f43c3c994181a841f52d2dccac07, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/39f0f19c475f46db953f552b58a804ab, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8d320e0f206c4773adc6b9aaa4bad1ce, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/5de757839dd44e29a1ce40c701b164d0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/19da1e97d21d4f57bfae6566a7769fc3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/fdbc49e0b5444f269f3b32c99aa45a74, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4e92d82ffcb6451cbbf3fb4dab1da527, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/b8175f11cb71488ca0289d20466902d4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/dcea1bddb6fe4883835c30b750daf916, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/13eb738909744176b23cee768a46953d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b9d130f826c4958ac592d06a8ad7887, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/194d93f8480f493db69722dfb6730ea4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d6e145ddba04404b94dcabefcf13f802, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d3f1a2de914346da8b13efbe2c2479ef, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/076a86cdf97141d992a991a42aea8309, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/10626efe63b04535a52d390cb7cf0833] to archive 2024-11-12T07:48:36,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:48:36,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d5d456b5595a44d89e04b3c94801190c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d5d456b5595a44d89e04b3c94801190c 2024-11-12T07:48:36,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/9196e3a1e6284bf4b7f2b5b911b39fc2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/9196e3a1e6284bf4b7f2b5b911b39fc2 2024-11-12T07:48:36,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/35608baa4ca449eabb7c4f8768ebf11a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/35608baa4ca449eabb7c4f8768ebf11a 2024-11-12T07:48:36,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/778268ae7a3246d2bcc63d93f7baa470 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/778268ae7a3246d2bcc63d93f7baa470 2024-11-12T07:48:36,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/c3747c43ebc14cf09ac8878360bef6f5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/c3747c43ebc14cf09ac8878360bef6f5 2024-11-12T07:48:36,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d7a290f01b0241d78f0cd8d1f6348796 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d7a290f01b0241d78f0cd8d1f6348796 2024-11-12T07:48:36,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/e070f43c3c994181a841f52d2dccac07 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/e070f43c3c994181a841f52d2dccac07 2024-11-12T07:48:36,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/39f0f19c475f46db953f552b58a804ab to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/39f0f19c475f46db953f552b58a804ab 2024-11-12T07:48:36,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8d320e0f206c4773adc6b9aaa4bad1ce to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8d320e0f206c4773adc6b9aaa4bad1ce 2024-11-12T07:48:36,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/5de757839dd44e29a1ce40c701b164d0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/5de757839dd44e29a1ce40c701b164d0 2024-11-12T07:48:36,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/19da1e97d21d4f57bfae6566a7769fc3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/19da1e97d21d4f57bfae6566a7769fc3 2024-11-12T07:48:36,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/fdbc49e0b5444f269f3b32c99aa45a74 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/fdbc49e0b5444f269f3b32c99aa45a74 2024-11-12T07:48:36,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4e92d82ffcb6451cbbf3fb4dab1da527 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4e92d82ffcb6451cbbf3fb4dab1da527 2024-11-12T07:48:36,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/b8175f11cb71488ca0289d20466902d4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/b8175f11cb71488ca0289d20466902d4 2024-11-12T07:48:36,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/dcea1bddb6fe4883835c30b750daf916 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/dcea1bddb6fe4883835c30b750daf916 2024-11-12T07:48:36,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/13eb738909744176b23cee768a46953d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/13eb738909744176b23cee768a46953d 2024-11-12T07:48:36,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b9d130f826c4958ac592d06a8ad7887 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b9d130f826c4958ac592d06a8ad7887 2024-11-12T07:48:36,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/194d93f8480f493db69722dfb6730ea4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/194d93f8480f493db69722dfb6730ea4 2024-11-12T07:48:36,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d6e145ddba04404b94dcabefcf13f802 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d6e145ddba04404b94dcabefcf13f802 2024-11-12T07:48:36,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d3f1a2de914346da8b13efbe2c2479ef to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/d3f1a2de914346da8b13efbe2c2479ef 2024-11-12T07:48:36,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/076a86cdf97141d992a991a42aea8309 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/076a86cdf97141d992a991a42aea8309 2024-11-12T07:48:36,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/10626efe63b04535a52d390cb7cf0833 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/10626efe63b04535a52d390cb7cf0833 2024-11-12T07:48:36,251 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=4 2024-11-12T07:48:36,252 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. 2024-11-12T07:48:36,252 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 299b057d313257fac3b10771c8ca0f20: Waiting for close lock at 1731397714480Running coprocessor pre-close hooks at 1731397714480Disabling compacts and flushes for region at 1731397714480Disabling writes for close at 1731397714480Obtaining lock to block concurrent updates at 1731397714481 (+1 ms)Preparing flush snapshotting stores in 299b057d313257fac3b10771c8ca0f20 at 1731397714481Finished memstore snapshotting TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731397714481Flushing stores of TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20. at 1731397714482 (+1 ms)Flushing 299b057d313257fac3b10771c8ca0f20/A: creating writer at 1731397714482Flushing mob file 299b057d313257fac3b10771c8ca0f20/A: appending metadata at 1731397714491 (+9 ms)Flushing mob file 299b057d313257fac3b10771c8ca0f20/A: closing flushed file at 1731397714491Flushing 299b057d313257fac3b10771c8ca0f20/A: appending metadata at 1731397714909 (+418 ms)Flushing 299b057d313257fac3b10771c8ca0f20/A: closing flushed file at 1731397714909Flushing 299b057d313257fac3b10771c8ca0f20/B: creating writer at 1731397715322 (+413 ms)Flushing 299b057d313257fac3b10771c8ca0f20/B: appending metadata at 1731397715327 (+5 ms)Flushing 299b057d313257fac3b10771c8ca0f20/B: closing flushed file at 1731397715327Flushing 299b057d313257fac3b10771c8ca0f20/C: creating writer at 1731397715743 (+416 ms)Flushing 299b057d313257fac3b10771c8ca0f20/C: appending metadata at 1731397715746 (+3 ms)Flushing 299b057d313257fac3b10771c8ca0f20/C: closing flushed file at 1731397715746Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f1cd32e: reopening flushed file at 1731397716162 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48163b7b: reopening flushed file at 1731397716169 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@248b3786: reopening flushed file at 1731397716175 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 299b057d313257fac3b10771c8ca0f20 in 1700ms, sequenceid=340, compaction requested=true at 1731397716181 (+6 ms)Writing region close event to WAL at 1731397716249 (+68 ms)Running coprocessor post-close hooks at 1731397716252 (+3 ms)Closed at 1731397716252 2024-11-12T07:48:36,254 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,254 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=299b057d313257fac3b10771c8ca0f20, regionState=CLOSED 2024-11-12T07:48:36,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:48:36,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T07:48:36,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 299b057d313257fac3b10771c8ca0f20, server=f9fc96254ef8,44299,1731397650797 in 1.9280 sec 2024-11-12T07:48:36,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-12T07:48:36,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=299b057d313257fac3b10771c8ca0f20, UNASSIGN in 1.9330 sec 2024-11-12T07:48:36,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T07:48:36,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9360 sec 2024-11-12T07:48:36,262 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397716261"}]},"ts":"1731397716261"} 2024-11-12T07:48:36,263 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T07:48:36,263 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T07:48:36,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9470 sec 2024-11-12T07:48:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T07:48:36,469 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:36,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T07:48:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:36,475 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T07:48:36,476 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:36,481 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,486 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/recovered.edits] 2024-11-12T07:48:36,491 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/18df1ef01f614c09a2ddf447a659ea5c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/18df1ef01f614c09a2ddf447a659ea5c 2024-11-12T07:48:36,494 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/2bb769d2299e484ebd44b31b1a1bc5e4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/2bb769d2299e484ebd44b31b1a1bc5e4 2024-11-12T07:48:36,496 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/722480ac826844b282ceb7b8e4fa14d0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/A/722480ac826844b282ceb7b8e4fa14d0 2024-11-12T07:48:36,501 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0eeb7c50a73f48899a54a491e8abc945 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/0eeb7c50a73f48899a54a491e8abc945 2024-11-12T07:48:36,503 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/47ceaf92dff94ff896b0d4b71330f875 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/47ceaf92dff94ff896b0d4b71330f875 2024-11-12T07:48:36,505 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/730b46c9a59148a192cdf36d0233b027 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/B/730b46c9a59148a192cdf36d0233b027 2024-11-12T07:48:36,509 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/07a7446455cd4eeca003ca5da29125ef to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/07a7446455cd4eeca003ca5da29125ef 2024-11-12T07:48:36,511 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4695ce505c7c484daaa6a0ef69c0ebd4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/4695ce505c7c484daaa6a0ef69c0ebd4 2024-11-12T07:48:36,513 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b7edf2e638c40cba0da0e117b2823a4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/C/8b7edf2e638c40cba0da0e117b2823a4 2024-11-12T07:48:36,515 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/recovered.edits/343.seqid to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20/recovered.edits/343.seqid 2024-11-12T07:48:36,516 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,516 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T07:48:36,516 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T07:48:36,517 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T07:48:36,520 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121ab92b05d9334b17b64a551cdc5ce7d4_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121ab92b05d9334b17b64a551cdc5ce7d4_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,521 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111230e57fab68df488b87bc7fde0b127b45_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111230e57fab68df488b87bc7fde0b127b45_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,522 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123fa92756968949898abcdb8d4759be22_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123fa92756968949898abcdb8d4759be22_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,523 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111242d43af324a749eea53e0e091c36140d_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111242d43af324a749eea53e0e091c36140d_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,524 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d32e1f24f49488f8767b4727d248e0e_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d32e1f24f49488f8767b4727d248e0e_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,525 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127cf9ec4960f8440c87c90f7cb5d386c2_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127cf9ec4960f8440c87c90f7cb5d386c2_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,526 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127d23fa86319d4b17af2f4c7bb092515f_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127d23fa86319d4b17af2f4c7bb092515f_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,527 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293b5f32ce50a43e48524ad34bce45ba3_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293b5f32ce50a43e48524ad34bce45ba3_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,528 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112987368e281ad4ba1955904601e68a517_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112987368e281ad4ba1955904601e68a517_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,529 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a0246e30a4954b3085fa3b28924aaedf_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a0246e30a4954b3085fa3b28924aaedf_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,530 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ae668a17aead422499c4c99a0dac3f28_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ae668a17aead422499c4c99a0dac3f28_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,531 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b504018378de43aca7502bdc615f0d82_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b504018378de43aca7502bdc615f0d82_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,532 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c15cdbc1df4240e2be3055bfe02af5f7_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c15cdbc1df4240e2be3055bfe02af5f7_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,533 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ca5eebf693be4498b43e1eab36130695_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ca5eebf693be4498b43e1eab36130695_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,535 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d40cd1e6cd844eb4bdbdc97c3dc9021c_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d40cd1e6cd844eb4bdbdc97c3dc9021c_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,536 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112da54d584b1d74ca885c1733cb328cc04_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112da54d584b1d74ca885c1733cb328cc04_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,537 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e47b1f04ffa5401b9c4ba2ad4fb62e4a_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e47b1f04ffa5401b9c4ba2ad4fb62e4a_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,538 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eb9b21b6b10b4000981060637b0b6c1d_299b057d313257fac3b10771c8ca0f20 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112eb9b21b6b10b4000981060637b0b6c1d_299b057d313257fac3b10771c8ca0f20 2024-11-12T07:48:36,539 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T07:48:36,540 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:36,543 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T07:48:36,545 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T07:48:36,546 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:36,546 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T07:48:36,546 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731397716546"}]},"ts":"9223372036854775807"} 2024-11-12T07:48:36,548 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T07:48:36,548 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 299b057d313257fac3b10771c8ca0f20, NAME => 'TestAcidGuarantees,,1731397683122.299b057d313257fac3b10771c8ca0f20.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T07:48:36,548 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T07:48:36,548 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731397716548"}]},"ts":"9223372036854775807"} 2024-11-12T07:48:36,550 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T07:48:36,550 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:36,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-12T07:48:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T07:48:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T07:48:36,578 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:36,589 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=447 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=432 (was 366) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5990 (was 6054) 2024-11-12T07:48:36,600 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=432, ProcessCount=11, AvailableMemoryMB=5990 2024-11-12T07:48:36,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:48:36,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:48:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T07:48:36,604 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:48:36,604 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:36,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-12T07:48:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T07:48:36,605 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:48:36,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742060_1236 (size=1061) 2024-11-12T07:48:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T07:48:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T07:48:37,018 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f 2024-11-12T07:48:37,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742061_1237 (size=53) 2024-11-12T07:48:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T07:48:37,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:48:37,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a03352d0297c5ba80936c4f6927633e5, disabling compactions & flushes 2024-11-12T07:48:37,432 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:37,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:37,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. after waiting 0 ms 2024-11-12T07:48:37,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:37,432 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:37,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a03352d0297c5ba80936c4f6927633e5: Waiting for close lock at 1731397717431Disabling compacts and flushes for region at 1731397717431Disabling writes for close at 1731397717432 (+1 ms)Writing region close event to WAL at 1731397717432Closed at 1731397717432 2024-11-12T07:48:37,435 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:48:37,435 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731397717435"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397717435"}]},"ts":"1731397717435"} 2024-11-12T07:48:37,438 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:48:37,439 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:48:37,440 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397717439"}]},"ts":"1731397717439"} 2024-11-12T07:48:37,442 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T07:48:37,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, ASSIGN}] 2024-11-12T07:48:37,444 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, ASSIGN 2024-11-12T07:48:37,445 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, ASSIGN; state=OFFLINE, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=false 2024-11-12T07:48:37,596 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a03352d0297c5ba80936c4f6927633e5, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:37,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, ASSIGN because future has completed 2024-11-12T07:48:37,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:48:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T07:48:37,762 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:37,762 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:48:37,763 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,763 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:48:37,763 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,763 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,765 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,767 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:37,767 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03352d0297c5ba80936c4f6927633e5 columnFamilyName A 2024-11-12T07:48:37,767 DEBUG [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:37,768 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.HStore(327): Store=a03352d0297c5ba80936c4f6927633e5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:37,768 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,770 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:37,770 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03352d0297c5ba80936c4f6927633e5 columnFamilyName B 2024-11-12T07:48:37,771 DEBUG [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:37,771 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.HStore(327): Store=a03352d0297c5ba80936c4f6927633e5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:37,771 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,773 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:48:37,773 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a03352d0297c5ba80936c4f6927633e5 columnFamilyName C 2024-11-12T07:48:37,773 DEBUG [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:48:37,774 INFO [StoreOpener-a03352d0297c5ba80936c4f6927633e5-1 {}] regionserver.HStore(327): Store=a03352d0297c5ba80936c4f6927633e5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:48:37,775 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:37,775 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,776 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,776 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,777 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,777 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,778 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:48:37,779 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,781 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:48:37,781 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened a03352d0297c5ba80936c4f6927633e5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66234766, jitterRate=-0.013025075197219849}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:48:37,781 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:37,782 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for a03352d0297c5ba80936c4f6927633e5: Running coprocessor pre-open hook at 1731397717763Writing region info on filesystem at 1731397717763Initializing all the Stores at 1731397717764 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397717764Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397717765 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397717765Cleaning up temporary data from old regions at 1731397717777 (+12 ms)Running coprocessor post-open hooks at 1731397717782 (+5 ms)Region opened successfully at 1731397717782 2024-11-12T07:48:37,783 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., pid=61, masterSystemTime=1731397717754 2024-11-12T07:48:37,785 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:37,785 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:37,785 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a03352d0297c5ba80936c4f6927633e5, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:37,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:48:37,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T07:48:37,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 in 188 msec 2024-11-12T07:48:37,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-12T07:48:37,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, ASSIGN in 348 msec 2024-11-12T07:48:37,793 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:48:37,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397717793"}]},"ts":"1731397717793"} 2024-11-12T07:48:37,795 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T07:48:37,796 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:48:37,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1940 sec 2024-11-12T07:48:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-12T07:48:38,749 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:38,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,756 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3090382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1debf58d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,763 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:48:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,43319,1731397650023 2024-11-12T07:48:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27accce3 2024-11-12T07:48:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:48:38,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:48:38,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503d4dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b4b11f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,771 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7a7a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,772 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710ae69b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,774 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f98a1f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,776 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7904eb33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@561759fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,781 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56421a49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,783 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ee11caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,784 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e6710ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,787 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c31fab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,788 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17308975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,791 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7757ae39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c10b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,794 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d509c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,795 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344ac1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,797 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b917ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b180924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:48:38,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:48:38,800 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:48:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:48:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:48:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300bac5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:48:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:48:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:38,801 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:48:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8278bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:48:38,802 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,802 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,802 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,802 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,802 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:38,803 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,803 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,803 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,804 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,804 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,804 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:48:38,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,805 DEBUG [Thread-1108 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:48:38,806 DEBUG [Thread-1108 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:48:38,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:48:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:48:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-12T07:48:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T07:48:38,854 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:38,855 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:38,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:38,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:38,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:48:38,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:38,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:38,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:38,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:38,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:38,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:38,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731397718903, max timestamp=1731397718912Num uniques 859; 2024-11-12T07:48:38,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e84bfc879bd64ee6a2d07c2ae1f960fc is 50, key is test_row_0/A:col10/1731397718854/Put/seqid=0 2024-11-12T07:48:38,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731397718903, max timestamp=1731397718912Num uniques 859; 2024-11-12T07:48:38,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T07:48:38,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T07:48:38,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731397718903, max timestamp=1731397718912Num uniques 859; 2024-11-12T07:48:38,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T07:48:38,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742062_1238 (size=16681) 2024-11-12T07:48:38,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e84bfc879bd64ee6a2d07c2ae1f960fc 2024-11-12T07:48:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397778934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:38,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397778935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397778934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397778939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397778941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:38,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/22c4151c003847068a5d69a59caaecd7 is 50, key is test_row_0/B:col10/1731397718854/Put/seqid=0 2024-11-12T07:48:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T07:48:38,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742063_1239 (size=12001) 2024-11-12T07:48:38,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/22c4151c003847068a5d69a59caaecd7 2024-11-12T07:48:39,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T07:48:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:39,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:39,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:39,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/039daa3b35f442ec857839427342ed89 is 50, key is test_row_0/C:col10/1731397718854/Put/seqid=0 2024-11-12T07:48:39,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742064_1240 (size=12001) 2024-11-12T07:48:39,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/039daa3b35f442ec857839427342ed89 2024-11-12T07:48:39,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397779048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e84bfc879bd64ee6a2d07c2ae1f960fc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e84bfc879bd64ee6a2d07c2ae1f960fc 2024-11-12T07:48:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397779048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397779048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e84bfc879bd64ee6a2d07c2ae1f960fc, entries=250, sequenceid=12, filesize=16.3 K 2024-11-12T07:48:39,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/22c4151c003847068a5d69a59caaecd7 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/22c4151c003847068a5d69a59caaecd7 2024-11-12T07:48:39,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/22c4151c003847068a5d69a59caaecd7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T07:48:39,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/039daa3b35f442ec857839427342ed89 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/039daa3b35f442ec857839427342ed89 2024-11-12T07:48:39,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397779058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397779058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/039daa3b35f442ec857839427342ed89, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T07:48:39,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a03352d0297c5ba80936c4f6927633e5 in 177ms, sequenceid=12, compaction requested=false 2024-11-12T07:48:39,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:39,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T07:48:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:39,162 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-12T07:48:39,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:48:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T07:48:39,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/755ea9b94f5746b796aef480acee6e19 is 50, key is test_row_0/A:col10/1731397718940/Put/seqid=0 2024-11-12T07:48:39,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742065_1241 (size=12001) 2024-11-12T07:48:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:39,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:39,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397779267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397779267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397779270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397779274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397779274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397779377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397779388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397779388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T07:48:39,575 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/755ea9b94f5746b796aef480acee6e19 2024-11-12T07:48:39,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397779577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397779577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/d76e6adb5e4a464c89eac6a7cc5c7460 is 50, key is test_row_0/B:col10/1731397718940/Put/seqid=0 2024-11-12T07:48:39,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742066_1242 (size=12001) 2024-11-12T07:48:39,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397779588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397779598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397779598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397779897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397779907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397779908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T07:48:39,989 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/d76e6adb5e4a464c89eac6a7cc5c7460 2024-11-12T07:48:39,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/9288ff7d568049f0bd4526468b6c6f79 is 50, key is test_row_0/C:col10/1731397718940/Put/seqid=0 2024-11-12T07:48:40,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742067_1243 (size=12001) 2024-11-12T07:48:40,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397780088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:40,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397780088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:40,402 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/9288ff7d568049f0bd4526468b6c6f79 2024-11-12T07:48:40,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/755ea9b94f5746b796aef480acee6e19 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/755ea9b94f5746b796aef480acee6e19 2024-11-12T07:48:40,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397780407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:40,411 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/755ea9b94f5746b796aef480acee6e19, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T07:48:40,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/d76e6adb5e4a464c89eac6a7cc5c7460 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/d76e6adb5e4a464c89eac6a7cc5c7460 2024-11-12T07:48:40,416 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/d76e6adb5e4a464c89eac6a7cc5c7460, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T07:48:40,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/9288ff7d568049f0bd4526468b6c6f79 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/9288ff7d568049f0bd4526468b6c6f79 2024-11-12T07:48:40,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397780418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397780418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:40,422 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/9288ff7d568049f0bd4526468b6c6f79, entries=150, sequenceid=46, filesize=11.7 K 2024-11-12T07:48:40,423 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for a03352d0297c5ba80936c4f6927633e5 in 1261ms, sequenceid=46, compaction requested=false 2024-11-12T07:48:40,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:40,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:40,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-12T07:48:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-12T07:48:40,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T07:48:40,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5700 sec 2024-11-12T07:48:40,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.6270 sec 2024-11-12T07:48:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T07:48:40,998 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:40,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-12T07:48:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T07:48:41,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:41,001 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:41,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:41,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T07:48:41,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:41,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:41,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:41,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:41,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:41,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:41,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/56e8f109773046f0be5d9ee0faa625b8 is 50, key is test_row_0/A:col10/1731397719273/Put/seqid=0 2024-11-12T07:48:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T07:48:41,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742068_1244 (size=14341) 2024-11-12T07:48:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397781138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397781139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T07:48:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397781248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397781248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T07:48:41,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:41,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T07:48:41,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397781418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,420 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in a03352d0297c5ba80936c4f6927633e5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397781438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,440 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in a03352d0297c5ba80936c4f6927633e5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397781438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,441 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in a03352d0297c5ba80936c4f6927633e5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T07:48:41,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:41,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397781457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397781457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/56e8f109773046f0be5d9ee0faa625b8 2024-11-12T07:48:41,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/3825f9fd215d417ab861ca57c09119f6 is 50, key is test_row_0/B:col10/1731397719273/Put/seqid=0 2024-11-12T07:48:41,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742069_1245 (size=12001) 2024-11-12T07:48:41,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/3825f9fd215d417ab861ca57c09119f6 2024-11-12T07:48:41,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/dc51a03a7c2e46048c3f1e9f51a1a816 is 50, key is test_row_0/C:col10/1731397719273/Put/seqid=0 2024-11-12T07:48:41,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742070_1246 (size=12001) 2024-11-12T07:48:41,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T07:48:41,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:41,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T07:48:41,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T07:48:41,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:41,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397781768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397781768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:41,777 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:48:41,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T07:48:41,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:41,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:41,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/dc51a03a7c2e46048c3f1e9f51a1a816 2024-11-12T07:48:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/56e8f109773046f0be5d9ee0faa625b8 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/56e8f109773046f0be5d9ee0faa625b8 2024-11-12T07:48:41,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/56e8f109773046f0be5d9ee0faa625b8, entries=200, sequenceid=59, filesize=14.0 K 2024-11-12T07:48:41,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/3825f9fd215d417ab861ca57c09119f6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/3825f9fd215d417ab861ca57c09119f6 2024-11-12T07:48:41,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/3825f9fd215d417ab861ca57c09119f6, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T07:48:41,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/dc51a03a7c2e46048c3f1e9f51a1a816 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/dc51a03a7c2e46048c3f1e9f51a1a816 2024-11-12T07:48:41,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/dc51a03a7c2e46048c3f1e9f51a1a816, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T07:48:41,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a03352d0297c5ba80936c4f6927633e5 in 869ms, sequenceid=59, compaction requested=true 2024-11-12T07:48:41,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:41,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:41,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:41,970 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:41,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:41,970 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:41,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:41,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:41,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:41,971 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:41,971 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:41,971 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/A is initiating minor compaction (all files) 2024-11-12T07:48:41,971 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/B is initiating minor compaction (all files) 2024-11-12T07:48:41,971 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/A in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,971 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/B in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,971 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/22c4151c003847068a5d69a59caaecd7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/d76e6adb5e4a464c89eac6a7cc5c7460, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/3825f9fd215d417ab861ca57c09119f6] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.2 K 2024-11-12T07:48:41,971 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e84bfc879bd64ee6a2d07c2ae1f960fc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/755ea9b94f5746b796aef480acee6e19, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/56e8f109773046f0be5d9ee0faa625b8] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=42.0 K 2024-11-12T07:48:41,971 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e84bfc879bd64ee6a2d07c2ae1f960fc, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731397718810 2024-11-12T07:48:41,971 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c4151c003847068a5d69a59caaecd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731397718854 2024-11-12T07:48:41,972 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 755ea9b94f5746b796aef480acee6e19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731397718932 2024-11-12T07:48:41,972 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d76e6adb5e4a464c89eac6a7cc5c7460, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731397718932 2024-11-12T07:48:41,972 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 3825f9fd215d417ab861ca57c09119f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731397719264 2024-11-12T07:48:41,972 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56e8f109773046f0be5d9ee0faa625b8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731397719264 2024-11-12T07:48:41,978 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:41,979 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/b1808687e04846d99473b31020af69cf is 50, key is test_row_0/B:col10/1731397719273/Put/seqid=0 2024-11-12T07:48:41,981 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#A#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:41,981 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/516e51f4557c4e8a9ce90dd56aa4c311 is 50, key is test_row_0/A:col10/1731397719273/Put/seqid=0 2024-11-12T07:48:41,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742071_1247 (size=12104) 2024-11-12T07:48:41,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742072_1248 (size=12104) 2024-11-12T07:48:41,989 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/b1808687e04846d99473b31020af69cf as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/b1808687e04846d99473b31020af69cf 2024-11-12T07:48:41,992 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/516e51f4557c4e8a9ce90dd56aa4c311 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/516e51f4557c4e8a9ce90dd56aa4c311 2024-11-12T07:48:41,994 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/B of a03352d0297c5ba80936c4f6927633e5 into b1808687e04846d99473b31020af69cf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:41,994 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:41,994 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/B, priority=13, startTime=1731397721970; duration=0sec 2024-11-12T07:48:41,994 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:41,994 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:B 2024-11-12T07:48:41,994 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:41,995 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:41,995 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/C is initiating minor compaction (all files) 2024-11-12T07:48:41,995 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/C in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:41,995 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/039daa3b35f442ec857839427342ed89, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/9288ff7d568049f0bd4526468b6c6f79, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/dc51a03a7c2e46048c3f1e9f51a1a816] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.2 K 2024-11-12T07:48:41,996 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 039daa3b35f442ec857839427342ed89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731397718854 2024-11-12T07:48:41,996 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9288ff7d568049f0bd4526468b6c6f79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731397718932 2024-11-12T07:48:41,996 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting dc51a03a7c2e46048c3f1e9f51a1a816, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731397719264 2024-11-12T07:48:41,998 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/A of a03352d0297c5ba80936c4f6927633e5 into 516e51f4557c4e8a9ce90dd56aa4c311(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:41,998 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:41,998 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/A, priority=13, startTime=1731397721970; duration=0sec 2024-11-12T07:48:41,998 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:41,998 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:A 2024-11-12T07:48:42,005 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:42,006 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/06162777cd9a4cad9c6c3a0410ef4c26 is 50, key is test_row_0/C:col10/1731397719273/Put/seqid=0 2024-11-12T07:48:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742073_1249 (size=12104) 2024-11-12T07:48:42,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T07:48:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:42,070 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T07:48:42,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:42,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:42,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:42,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:42,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:42,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:42,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7e94a43f37b940938aabc1f2afd14fd9 is 50, key is test_row_0/A:col10/1731397721137/Put/seqid=0 2024-11-12T07:48:42,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742074_1250 (size=12001) 2024-11-12T07:48:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T07:48:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:42,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:42,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397782293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:42,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:42,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397782296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:42,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:42,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397782398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:42,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397782408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:42,418 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/06162777cd9a4cad9c6c3a0410ef4c26 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/06162777cd9a4cad9c6c3a0410ef4c26 2024-11-12T07:48:42,423 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/C of a03352d0297c5ba80936c4f6927633e5 into 06162777cd9a4cad9c6c3a0410ef4c26(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:42,423 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:42,423 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/C, priority=13, startTime=1731397721970; duration=0sec 2024-11-12T07:48:42,423 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:42,423 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:C 2024-11-12T07:48:42,482 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7e94a43f37b940938aabc1f2afd14fd9 2024-11-12T07:48:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/173ae990a67a48018b7c0e05b8f051cc is 50, key is test_row_0/B:col10/1731397721137/Put/seqid=0 2024-11-12T07:48:42,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742075_1251 (size=12001) 2024-11-12T07:48:42,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397782607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397782618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:42,896 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/173ae990a67a48018b7c0e05b8f051cc 2024-11-12T07:48:42,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/99abf850640d45178c0fe0d9ed920669 is 50, key is test_row_0/C:col10/1731397721137/Put/seqid=0 2024-11-12T07:48:42,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742076_1252 (size=12001) 2024-11-12T07:48:42,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397782918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:42,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397782928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T07:48:43,307 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/99abf850640d45178c0fe0d9ed920669 2024-11-12T07:48:43,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7e94a43f37b940938aabc1f2afd14fd9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7e94a43f37b940938aabc1f2afd14fd9 2024-11-12T07:48:43,314 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7e94a43f37b940938aabc1f2afd14fd9, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T07:48:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/173ae990a67a48018b7c0e05b8f051cc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/173ae990a67a48018b7c0e05b8f051cc 2024-11-12T07:48:43,318 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/173ae990a67a48018b7c0e05b8f051cc, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T07:48:43,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/99abf850640d45178c0fe0d9ed920669 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/99abf850640d45178c0fe0d9ed920669 2024-11-12T07:48:43,323 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/99abf850640d45178c0fe0d9ed920669, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T07:48:43,323 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a03352d0297c5ba80936c4f6927633e5 in 1253ms, sequenceid=80, compaction requested=false 2024-11-12T07:48:43,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-12T07:48:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-12T07:48:43,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T07:48:43,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-11-12T07:48:43,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.3290 sec 2024-11-12T07:48:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:43,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:48:43,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:43,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:43,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:43,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:43,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:43,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:43,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/4bbb2672f00c4b31ac6ed662495c509e is 50, key is test_row_0/A:col10/1731397722294/Put/seqid=0 2024-11-12T07:48:43,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742077_1253 (size=14341) 2024-11-12T07:48:43,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397783457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397783458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397783459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,464 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in a03352d0297c5ba80936c4f6927633e5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397783461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397783468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,471 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in a03352d0297c5ba80936c4f6927633e5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397783568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397783568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397783568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397783778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397783778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397783778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:43,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:43,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/4bbb2672f00c4b31ac6ed662495c509e 2024-11-12T07:48:43,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a50d89c502af4e88a27984ffaf041623 is 50, key is test_row_0/B:col10/1731397722294/Put/seqid=0 2024-11-12T07:48:43,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742078_1254 (size=12001) 2024-11-12T07:48:44,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397784087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397784087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397784088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:44,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a50d89c502af4e88a27984ffaf041623 2024-11-12T07:48:44,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/199516896373432c998cf0faa1ad651d is 50, key is test_row_0/C:col10/1731397722294/Put/seqid=0 2024-11-12T07:48:44,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742079_1255 (size=12001) 2024-11-12T07:48:44,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397784598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397784598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:44,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397784598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:44,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/199516896373432c998cf0faa1ad651d 2024-11-12T07:48:44,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/4bbb2672f00c4b31ac6ed662495c509e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/4bbb2672f00c4b31ac6ed662495c509e 2024-11-12T07:48:44,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/4bbb2672f00c4b31ac6ed662495c509e, entries=200, sequenceid=95, filesize=14.0 K 2024-11-12T07:48:44,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a50d89c502af4e88a27984ffaf041623 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a50d89c502af4e88a27984ffaf041623 2024-11-12T07:48:44,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a50d89c502af4e88a27984ffaf041623, entries=150, sequenceid=95, filesize=11.7 K 2024-11-12T07:48:44,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/199516896373432c998cf0faa1ad651d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/199516896373432c998cf0faa1ad651d 2024-11-12T07:48:44,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/199516896373432c998cf0faa1ad651d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-12T07:48:44,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a03352d0297c5ba80936c4f6927633e5 in 1298ms, sequenceid=95, compaction requested=true 2024-11-12T07:48:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:44,730 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:44,730 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:44,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:44,731 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:44,731 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:44,731 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/A is initiating minor compaction (all files) 2024-11-12T07:48:44,731 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/B is initiating minor compaction (all files) 2024-11-12T07:48:44,731 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/A in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:44,731 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/B in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:44,731 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/b1808687e04846d99473b31020af69cf, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/173ae990a67a48018b7c0e05b8f051cc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a50d89c502af4e88a27984ffaf041623] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.3 K 2024-11-12T07:48:44,731 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/516e51f4557c4e8a9ce90dd56aa4c311, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7e94a43f37b940938aabc1f2afd14fd9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/4bbb2672f00c4b31ac6ed662495c509e] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=37.5 K 2024-11-12T07:48:44,731 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting b1808687e04846d99473b31020af69cf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731397719264 2024-11-12T07:48:44,731 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516e51f4557c4e8a9ce90dd56aa4c311, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731397719264 2024-11-12T07:48:44,732 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e94a43f37b940938aabc1f2afd14fd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731397721123 2024-11-12T07:48:44,732 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 173ae990a67a48018b7c0e05b8f051cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731397721123 2024-11-12T07:48:44,732 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bbb2672f00c4b31ac6ed662495c509e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731397722278 2024-11-12T07:48:44,732 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting a50d89c502af4e88a27984ffaf041623, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731397722278 2024-11-12T07:48:44,738 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:44,739 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7848afc337124f799ec64ff56a808719 is 50, key is test_row_0/A:col10/1731397722294/Put/seqid=0 2024-11-12T07:48:44,742 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:44,743 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/c5da6055396546059c8603d4f82828ad is 50, key is test_row_0/B:col10/1731397722294/Put/seqid=0 2024-11-12T07:48:44,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742081_1257 (size=12207) 2024-11-12T07:48:44,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742080_1256 (size=12207) 2024-11-12T07:48:44,776 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/c5da6055396546059c8603d4f82828ad as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c5da6055396546059c8603d4f82828ad 2024-11-12T07:48:44,782 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/B of a03352d0297c5ba80936c4f6927633e5 into c5da6055396546059c8603d4f82828ad(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:44,782 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:44,782 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/B, priority=13, startTime=1731397724730; duration=0sec 2024-11-12T07:48:44,782 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:44,782 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:B 2024-11-12T07:48:44,782 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:44,783 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:44,783 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/C is initiating minor compaction (all files) 2024-11-12T07:48:44,783 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/C in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:44,783 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/06162777cd9a4cad9c6c3a0410ef4c26, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/99abf850640d45178c0fe0d9ed920669, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/199516896373432c998cf0faa1ad651d] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.3 K 2024-11-12T07:48:44,783 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 06162777cd9a4cad9c6c3a0410ef4c26, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731397719264 2024-11-12T07:48:44,784 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 99abf850640d45178c0fe0d9ed920669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731397721123 2024-11-12T07:48:44,784 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 199516896373432c998cf0faa1ad651d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731397722278 2024-11-12T07:48:44,794 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#C#compaction#215 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:44,794 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/5210b48b9d744ea28108199402a456b7 is 50, key is test_row_0/C:col10/1731397722294/Put/seqid=0 2024-11-12T07:48:44,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742082_1258 (size=12207) 2024-11-12T07:48:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T07:48:45,178 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:45,178 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7848afc337124f799ec64ff56a808719 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7848afc337124f799ec64ff56a808719 2024-11-12T07:48:45,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-12T07:48:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T07:48:45,181 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:45,182 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:45,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:45,184 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/A of a03352d0297c5ba80936c4f6927633e5 into 7848afc337124f799ec64ff56a808719(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:45,184 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:45,184 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/A, priority=13, startTime=1731397724730; duration=0sec 2024-11-12T07:48:45,184 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:45,184 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:A 2024-11-12T07:48:45,207 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/5210b48b9d744ea28108199402a456b7 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/5210b48b9d744ea28108199402a456b7 2024-11-12T07:48:45,211 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/C of a03352d0297c5ba80936c4f6927633e5 into 5210b48b9d744ea28108199402a456b7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:45,211 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:45,211 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/C, priority=13, startTime=1731397724730; duration=0sec 2024-11-12T07:48:45,211 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:45,211 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:C 2024-11-12T07:48:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T07:48:45,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T07:48:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:45,335 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:48:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:45,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:45,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/818010a0baa848b29df059e867cd1c32 is 50, key is test_row_0/A:col10/1731397723457/Put/seqid=0 2024-11-12T07:48:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742083_1259 (size=12001) 2024-11-12T07:48:45,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T07:48:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:45,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397785626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397785626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:45,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397785627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397785738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397785738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397785738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:45,751 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/818010a0baa848b29df059e867cd1c32 2024-11-12T07:48:45,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/83dc9860272d4fa8bdef595379a1b4f2 is 50, key is test_row_0/B:col10/1731397723457/Put/seqid=0 2024-11-12T07:48:45,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742084_1260 (size=12001) 2024-11-12T07:48:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T07:48:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397785948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397785948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397785948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,177 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/83dc9860272d4fa8bdef595379a1b4f2 2024-11-12T07:48:46,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/403a30631b99471c80077533777a510e is 50, key is test_row_0/C:col10/1731397723457/Put/seqid=0 2024-11-12T07:48:46,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742085_1261 (size=12001) 2024-11-12T07:48:46,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397786257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397786257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397786257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T07:48:46,588 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/403a30631b99471c80077533777a510e 2024-11-12T07:48:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/818010a0baa848b29df059e867cd1c32 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/818010a0baa848b29df059e867cd1c32 2024-11-12T07:48:46,596 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/818010a0baa848b29df059e867cd1c32, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T07:48:46,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/83dc9860272d4fa8bdef595379a1b4f2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/83dc9860272d4fa8bdef595379a1b4f2 2024-11-12T07:48:46,602 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/83dc9860272d4fa8bdef595379a1b4f2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T07:48:46,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/403a30631b99471c80077533777a510e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/403a30631b99471c80077533777a510e 2024-11-12T07:48:46,607 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/403a30631b99471c80077533777a510e, entries=150, sequenceid=116, filesize=11.7 K 2024-11-12T07:48:46,608 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a03352d0297c5ba80936c4f6927633e5 in 1273ms, sequenceid=116, compaction requested=false 2024-11-12T07:48:46,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:46,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:46,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-12T07:48:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-12T07:48:46,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-12T07:48:46,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-11-12T07:48:46,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4340 sec 2024-11-12T07:48:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:46,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T07:48:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:46,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/80be6d4c7dab473da7691dc2cadd8050 is 50, key is test_row_0/A:col10/1731397725626/Put/seqid=0 2024-11-12T07:48:46,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742086_1262 (size=14441) 2024-11-12T07:48:46,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397786789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397786791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397786794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397786898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397786898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:46,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:46,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397786908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397787107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397787108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397787118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/80be6d4c7dab473da7691dc2cadd8050 2024-11-12T07:48:47,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/099aa36bf6524574b059475c0d0266d6 is 50, key is test_row_0/B:col10/1731397725626/Put/seqid=0 2024-11-12T07:48:47,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742087_1263 (size=12101) 2024-11-12T07:48:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T07:48:47,328 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:47,328 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-12T07:48:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T07:48:47,330 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:47,330 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:47,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:47,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397787418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397787418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397787427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T07:48:47,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T07:48:47,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:47,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397787488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,492 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in a03352d0297c5ba80936c4f6927633e5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397787497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,500 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in a03352d0297c5ba80936c4f6927633e5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:48:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/099aa36bf6524574b059475c0d0266d6 2024-11-12T07:48:47,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/e20ce25f2d3346b0825364c9ab4a89e3 is 50, key is test_row_0/C:col10/1731397725626/Put/seqid=0 2024-11-12T07:48:47,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742088_1264 (size=12101) 2024-11-12T07:48:47,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/e20ce25f2d3346b0825364c9ab4a89e3 2024-11-12T07:48:47,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/80be6d4c7dab473da7691dc2cadd8050 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/80be6d4c7dab473da7691dc2cadd8050 2024-11-12T07:48:47,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/80be6d4c7dab473da7691dc2cadd8050, entries=200, sequenceid=131, filesize=14.1 K 2024-11-12T07:48:47,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/099aa36bf6524574b059475c0d0266d6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/099aa36bf6524574b059475c0d0266d6 2024-11-12T07:48:47,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/099aa36bf6524574b059475c0d0266d6, entries=150, sequenceid=131, filesize=11.8 K 2024-11-12T07:48:47,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/e20ce25f2d3346b0825364c9ab4a89e3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/e20ce25f2d3346b0825364c9ab4a89e3 2024-11-12T07:48:47,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/e20ce25f2d3346b0825364c9ab4a89e3, entries=150, sequenceid=131, filesize=11.8 K 2024-11-12T07:48:47,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a03352d0297c5ba80936c4f6927633e5 in 857ms, sequenceid=131, compaction requested=true 2024-11-12T07:48:47,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:47,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:47,627 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:47,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:47,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:47,627 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:47,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:47,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:47,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:47,628 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:47,628 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:47,628 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/B is initiating minor compaction (all files) 2024-11-12T07:48:47,628 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/A is initiating minor compaction (all files) 2024-11-12T07:48:47,628 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/A in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:47,628 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/B in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:47,629 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c5da6055396546059c8603d4f82828ad, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/83dc9860272d4fa8bdef595379a1b4f2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/099aa36bf6524574b059475c0d0266d6] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.5 K 2024-11-12T07:48:47,629 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7848afc337124f799ec64ff56a808719, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/818010a0baa848b29df059e867cd1c32, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/80be6d4c7dab473da7691dc2cadd8050] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=37.7 K 2024-11-12T07:48:47,629 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7848afc337124f799ec64ff56a808719, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731397722278 2024-11-12T07:48:47,629 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c5da6055396546059c8603d4f82828ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731397722278 2024-11-12T07:48:47,629 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 818010a0baa848b29df059e867cd1c32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731397723452 2024-11-12T07:48:47,629 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 83dc9860272d4fa8bdef595379a1b4f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731397723452 2024-11-12T07:48:47,630 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 099aa36bf6524574b059475c0d0266d6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731397725623 2024-11-12T07:48:47,630 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80be6d4c7dab473da7691dc2cadd8050, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731397725623 2024-11-12T07:48:47,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-12T07:48:47,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:47,636 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T07:48:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:47,637 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:47,637 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:47,637 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/53cb06ae67744b6b82ca0e91409d1a7a is 50, key is test_row_0/A:col10/1731397725626/Put/seqid=0 2024-11-12T07:48:47,637 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/92fe703125f04ea6982ae6a6b0193a71 is 50, key is test_row_0/B:col10/1731397725626/Put/seqid=0 2024-11-12T07:48:47,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/79f04a75b14a4c8c89ccd1e4670a7aa4 is 50, key is test_row_0/A:col10/1731397726788/Put/seqid=0 2024-11-12T07:48:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T07:48:47,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742089_1265 (size=12409) 2024-11-12T07:48:47,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742090_1266 (size=12409) 2024-11-12T07:48:47,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742091_1267 (size=12151) 2024-11-12T07:48:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:47,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:47,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397787945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397787947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397787948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:47,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T07:48:48,058 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/92fe703125f04ea6982ae6a6b0193a71 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/92fe703125f04ea6982ae6a6b0193a71 2024-11-12T07:48:48,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397788058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397788058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,064 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/B of a03352d0297c5ba80936c4f6927633e5 into 92fe703125f04ea6982ae6a6b0193a71(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:48,064 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:48,064 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/B, priority=13, startTime=1731397727627; duration=0sec 2024-11-12T07:48:48,064 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:48,064 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:B 2024-11-12T07:48:48,064 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:48,065 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:48,066 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/C is initiating minor compaction (all files) 2024-11-12T07:48:48,066 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/C in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:48,066 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/5210b48b9d744ea28108199402a456b7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/403a30631b99471c80077533777a510e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/e20ce25f2d3346b0825364c9ab4a89e3] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.5 K 2024-11-12T07:48:48,066 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/79f04a75b14a4c8c89ccd1e4670a7aa4 2024-11-12T07:48:48,066 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/53cb06ae67744b6b82ca0e91409d1a7a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/53cb06ae67744b6b82ca0e91409d1a7a 2024-11-12T07:48:48,067 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5210b48b9d744ea28108199402a456b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731397722278 2024-11-12T07:48:48,067 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 403a30631b99471c80077533777a510e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731397723452 2024-11-12T07:48:48,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397788064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,067 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e20ce25f2d3346b0825364c9ab4a89e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731397725623 2024-11-12T07:48:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,074 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/A of a03352d0297c5ba80936c4f6927633e5 into 53cb06ae67744b6b82ca0e91409d1a7a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:48,074 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:48,074 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/A, priority=13, startTime=1731397727627; duration=0sec 2024-11-12T07:48:48,074 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:48,074 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:A 2024-11-12T07:48:48,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/0b9a823bdeab452bb089959f54506a96 is 50, key is test_row_0/B:col10/1731397726788/Put/seqid=0 2024-11-12T07:48:48,077 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#C#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:48,077 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/69556b14be564885b5609bb21db450ff is 50, key is test_row_0/C:col10/1731397725626/Put/seqid=0 2024-11-12T07:48:48,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742093_1269 (size=12151) 2024-11-12T07:48:48,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742092_1268 (size=12409) 2024-11-12T07:48:48,099 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/69556b14be564885b5609bb21db450ff as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69556b14be564885b5609bb21db450ff 2024-11-12T07:48:48,104 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/C of a03352d0297c5ba80936c4f6927633e5 into 69556b14be564885b5609bb21db450ff(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:48,104 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:48,104 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/C, priority=13, startTime=1731397727627; duration=0sec 2024-11-12T07:48:48,104 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:48,104 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:C 2024-11-12T07:48:48,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397788268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397788268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397788277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T07:48:48,495 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/0b9a823bdeab452bb089959f54506a96 2024-11-12T07:48:48,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/f6ef1588b3dd4d63b8d9414810f3c280 is 50, key is test_row_0/C:col10/1731397726788/Put/seqid=0 2024-11-12T07:48:48,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742094_1270 (size=12151) 2024-11-12T07:48:48,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397788578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397788578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397788587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:48,905 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/f6ef1588b3dd4d63b8d9414810f3c280 2024-11-12T07:48:48,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/79f04a75b14a4c8c89ccd1e4670a7aa4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/79f04a75b14a4c8c89ccd1e4670a7aa4 2024-11-12T07:48:48,913 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/79f04a75b14a4c8c89ccd1e4670a7aa4, entries=150, sequenceid=148, filesize=11.9 K 2024-11-12T07:48:48,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/0b9a823bdeab452bb089959f54506a96 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/0b9a823bdeab452bb089959f54506a96 2024-11-12T07:48:48,917 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/0b9a823bdeab452bb089959f54506a96, entries=150, sequenceid=148, filesize=11.9 K 2024-11-12T07:48:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/f6ef1588b3dd4d63b8d9414810f3c280 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/f6ef1588b3dd4d63b8d9414810f3c280 2024-11-12T07:48:48,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/f6ef1588b3dd4d63b8d9414810f3c280, entries=150, sequenceid=148, filesize=11.9 K 2024-11-12T07:48:48,927 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a03352d0297c5ba80936c4f6927633e5 in 1291ms, sequenceid=148, compaction requested=false 2024-11-12T07:48:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-12T07:48:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-12T07:48:48,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T07:48:48,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6000 sec 2024-11-12T07:48:48,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6150 sec 2024-11-12T07:48:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:49,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T07:48:49,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:49,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:49,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:49,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:49,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:49,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:49,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/69b3d53c40914049b35abd5926808a16 is 50, key is test_row_0/A:col10/1731397729088/Put/seqid=0 2024-11-12T07:48:49,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742095_1271 (size=12151) 2024-11-12T07:48:49,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397789111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397789112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397789112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397789218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397789219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397789228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397789427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397789427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397789437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T07:48:49,478 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:49,479 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-12T07:48:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T07:48:49,480 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:49,481 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:49,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:49,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/69b3d53c40914049b35abd5926808a16 2024-11-12T07:48:49,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a61c68fbd10d4a9799a0ec7fab84b929 is 50, key is test_row_0/B:col10/1731397729088/Put/seqid=0 2024-11-12T07:48:49,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742096_1272 (size=12151) 2024-11-12T07:48:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T07:48:49,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T07:48:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:49,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:49,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397789738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397789738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397789747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:49,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T07:48:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:49,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:49,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T07:48:49,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a61c68fbd10d4a9799a0ec7fab84b929 2024-11-12T07:48:49,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/2a18a622a93f4bc78e7cd624ca2eda4e is 50, key is test_row_0/C:col10/1731397729088/Put/seqid=0 2024-11-12T07:48:49,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742097_1273 (size=12151) 2024-11-12T07:48:49,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T07:48:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:49,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:50,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T07:48:50,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:50,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:50,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T07:48:50,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T07:48:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:50,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:50,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:50,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:50,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:50,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397790247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:50,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397790248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:50,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397790258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:50,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/2a18a622a93f4bc78e7cd624ca2eda4e 2024-11-12T07:48:50,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/69b3d53c40914049b35abd5926808a16 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/69b3d53c40914049b35abd5926808a16 2024-11-12T07:48:50,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/69b3d53c40914049b35abd5926808a16, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T07:48:50,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a61c68fbd10d4a9799a0ec7fab84b929 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a61c68fbd10d4a9799a0ec7fab84b929 2024-11-12T07:48:50,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a61c68fbd10d4a9799a0ec7fab84b929, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T07:48:50,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/2a18a622a93f4bc78e7cd624ca2eda4e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/2a18a622a93f4bc78e7cd624ca2eda4e 2024-11-12T07:48:50,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/2a18a622a93f4bc78e7cd624ca2eda4e, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T07:48:50,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a03352d0297c5ba80936c4f6927633e5 in 1259ms, sequenceid=167, compaction requested=true 2024-11-12T07:48:50,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:50,349 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:50,349 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:50,350 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:50,350 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/B is initiating minor compaction (all files) 2024-11-12T07:48:50,350 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:50,350 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/A is initiating minor compaction (all files) 2024-11-12T07:48:50,350 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/B in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:50,350 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/92fe703125f04ea6982ae6a6b0193a71, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/0b9a823bdeab452bb089959f54506a96, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a61c68fbd10d4a9799a0ec7fab84b929] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.9 K 2024-11-12T07:48:50,350 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/A in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:50,350 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/53cb06ae67744b6b82ca0e91409d1a7a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/79f04a75b14a4c8c89ccd1e4670a7aa4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/69b3d53c40914049b35abd5926808a16] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.9 K 2024-11-12T07:48:50,351 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 92fe703125f04ea6982ae6a6b0193a71, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731397725623 2024-11-12T07:48:50,351 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53cb06ae67744b6b82ca0e91409d1a7a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731397725623 2024-11-12T07:48:50,351 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9a823bdeab452bb089959f54506a96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731397726780 2024-11-12T07:48:50,351 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f04a75b14a4c8c89ccd1e4670a7aa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731397726780 2024-11-12T07:48:50,351 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting a61c68fbd10d4a9799a0ec7fab84b929, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731397727941 2024-11-12T07:48:50,352 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69b3d53c40914049b35abd5926808a16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731397727941 2024-11-12T07:48:50,380 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:50,381 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/351c3b39c75843bab0bf9bfad01d4da9 is 50, key is test_row_0/B:col10/1731397729088/Put/seqid=0 2024-11-12T07:48:50,383 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:50,384 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7f83aef0c8ad4c669654e63963ca27c5 is 50, key is test_row_0/A:col10/1731397729088/Put/seqid=0 2024-11-12T07:48:50,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742098_1274 (size=12561) 2024-11-12T07:48:50,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-12T07:48:50,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:50,398 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T07:48:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:50,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742099_1275 (size=12561) 2024-11-12T07:48:50,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/3dc68ebc818b4f18b462e5c4db154c48 is 50, key is test_row_0/A:col10/1731397729111/Put/seqid=0 2024-11-12T07:48:50,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742100_1276 (size=12151) 2024-11-12T07:48:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T07:48:50,793 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/351c3b39c75843bab0bf9bfad01d4da9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/351c3b39c75843bab0bf9bfad01d4da9 2024-11-12T07:48:50,797 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/B of a03352d0297c5ba80936c4f6927633e5 into 351c3b39c75843bab0bf9bfad01d4da9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:50,797 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:50,797 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/B, priority=13, startTime=1731397730349; duration=0sec 2024-11-12T07:48:50,798 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:50,798 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:B 2024-11-12T07:48:50,798 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:50,799 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:50,799 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/C is initiating minor compaction (all files) 2024-11-12T07:48:50,799 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/C in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:50,799 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69556b14be564885b5609bb21db450ff, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/f6ef1588b3dd4d63b8d9414810f3c280, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/2a18a622a93f4bc78e7cd624ca2eda4e] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=35.9 K 2024-11-12T07:48:50,799 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 69556b14be564885b5609bb21db450ff, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731397725623 2024-11-12T07:48:50,800 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ef1588b3dd4d63b8d9414810f3c280, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731397726780 2024-11-12T07:48:50,800 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a18a622a93f4bc78e7cd624ca2eda4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731397727941 2024-11-12T07:48:50,802 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7f83aef0c8ad4c669654e63963ca27c5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7f83aef0c8ad4c669654e63963ca27c5 2024-11-12T07:48:50,807 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/3dc68ebc818b4f18b462e5c4db154c48 2024-11-12T07:48:50,807 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:50,808 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/221d1922501c4ac49c6f7fc83e507436 is 50, key is test_row_0/C:col10/1731397729088/Put/seqid=0 2024-11-12T07:48:50,811 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/A of a03352d0297c5ba80936c4f6927633e5 into 7f83aef0c8ad4c669654e63963ca27c5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:50,811 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:50,811 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/A, priority=13, startTime=1731397730348; duration=0sec 2024-11-12T07:48:50,811 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:50,811 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:A 2024-11-12T07:48:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/e37872be8c5247a088f7100b27a0dbce is 50, key is test_row_0/B:col10/1731397729111/Put/seqid=0 2024-11-12T07:48:50,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742101_1277 (size=12561) 2024-11-12T07:48:50,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742102_1278 (size=12151) 2024-11-12T07:48:50,849 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/e37872be8c5247a088f7100b27a0dbce 2024-11-12T07:48:50,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b3c18f749c784e248731c95cea6599c8 is 50, key is test_row_0/C:col10/1731397729111/Put/seqid=0 2024-11-12T07:48:50,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742103_1279 (size=12151) 2024-11-12T07:48:51,237 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/221d1922501c4ac49c6f7fc83e507436 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/221d1922501c4ac49c6f7fc83e507436 2024-11-12T07:48:51,241 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/C of a03352d0297c5ba80936c4f6927633e5 into 221d1922501c4ac49c6f7fc83e507436(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:51,242 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:51,242 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/C, priority=13, startTime=1731397730349; duration=0sec 2024-11-12T07:48:51,242 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:51,242 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:C 2024-11-12T07:48:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:51,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:51,261 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b3c18f749c784e248731c95cea6599c8 2024-11-12T07:48:51,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/3dc68ebc818b4f18b462e5c4db154c48 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/3dc68ebc818b4f18b462e5c4db154c48 2024-11-12T07:48:51,271 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/3dc68ebc818b4f18b462e5c4db154c48, entries=150, sequenceid=183, filesize=11.9 K 2024-11-12T07:48:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/e37872be8c5247a088f7100b27a0dbce as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/e37872be8c5247a088f7100b27a0dbce 2024-11-12T07:48:51,276 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/e37872be8c5247a088f7100b27a0dbce, entries=150, sequenceid=183, filesize=11.9 K 2024-11-12T07:48:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b3c18f749c784e248731c95cea6599c8 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b3c18f749c784e248731c95cea6599c8 2024-11-12T07:48:51,284 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b3c18f749c784e248731c95cea6599c8, entries=150, sequenceid=183, filesize=11.9 K 2024-11-12T07:48:51,285 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=73.80 KB/75570 for a03352d0297c5ba80936c4f6927633e5 in 887ms, sequenceid=183, compaction requested=false 2024-11-12T07:48:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:51,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-12T07:48:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:51,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T07:48:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-12T07:48:51,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:51,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:51,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:51,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:51,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:51,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:51,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/98518ebb24c24f1d8fb762239775f03e is 50, key is test_row_0/A:col10/1731397731284/Put/seqid=0 2024-11-12T07:48:51,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-12T07:48:51,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-11-12T07:48:51,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8150 sec 2024-11-12T07:48:51,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742104_1280 (size=14541) 2024-11-12T07:48:51,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397791341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397791342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397791342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397791448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397791458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397791458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-12T07:48:51,628 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:51,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-12T07:48:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T07:48:51,630 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:51,630 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:51,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:51,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397791658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397791668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397791668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/98518ebb24c24f1d8fb762239775f03e 2024-11-12T07:48:51,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/cb98d636c65644bab8099f307b1041c0 is 50, key is test_row_0/B:col10/1731397731284/Put/seqid=0 2024-11-12T07:48:51,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742105_1281 (size=12151) 2024-11-12T07:48:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T07:48:51,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T07:48:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:51,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:51,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:51,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T07:48:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:51,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T07:48:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397791968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397791978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:51,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397791978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:51,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:52,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T07:48:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/cb98d636c65644bab8099f307b1041c0 2024-11-12T07:48:52,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/d95ccb79a3aa4c77ad7ef4a2581f27c4 is 50, key is test_row_0/C:col10/1731397731284/Put/seqid=0 2024-11-12T07:48:52,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742106_1282 (size=12151) 2024-11-12T07:48:52,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T07:48:52,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:52,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T07:48:52,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T07:48:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397792478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:52,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397792488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:52,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:52,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397792488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:52,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T07:48:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:52,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:52,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/d95ccb79a3aa4c77ad7ef4a2581f27c4 2024-11-12T07:48:52,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/98518ebb24c24f1d8fb762239775f03e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/98518ebb24c24f1d8fb762239775f03e 2024-11-12T07:48:52,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/98518ebb24c24f1d8fb762239775f03e, entries=200, sequenceid=202, filesize=14.2 K 2024-11-12T07:48:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/cb98d636c65644bab8099f307b1041c0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/cb98d636c65644bab8099f307b1041c0 2024-11-12T07:48:52,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/cb98d636c65644bab8099f307b1041c0, entries=150, sequenceid=202, filesize=11.9 K 2024-11-12T07:48:52,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/d95ccb79a3aa4c77ad7ef4a2581f27c4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d95ccb79a3aa4c77ad7ef4a2581f27c4 2024-11-12T07:48:52,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d95ccb79a3aa4c77ad7ef4a2581f27c4, entries=150, sequenceid=202, filesize=11.9 K 2024-11-12T07:48:52,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a03352d0297c5ba80936c4f6927633e5 in 1284ms, sequenceid=202, compaction requested=true 2024-11-12T07:48:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:52,570 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:52,570 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:52,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:52,571 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:52,571 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:52,571 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/A is initiating minor compaction (all files) 2024-11-12T07:48:52,571 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/B is initiating minor compaction (all files) 2024-11-12T07:48:52,571 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/A in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,571 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/B in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,571 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7f83aef0c8ad4c669654e63963ca27c5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/3dc68ebc818b4f18b462e5c4db154c48, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/98518ebb24c24f1d8fb762239775f03e] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=38.3 K 2024-11-12T07:48:52,571 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/351c3b39c75843bab0bf9bfad01d4da9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/e37872be8c5247a088f7100b27a0dbce, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/cb98d636c65644bab8099f307b1041c0] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.0 K 2024-11-12T07:48:52,571 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 351c3b39c75843bab0bf9bfad01d4da9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731397727941 2024-11-12T07:48:52,571 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f83aef0c8ad4c669654e63963ca27c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731397727941 2024-11-12T07:48:52,572 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e37872be8c5247a088f7100b27a0dbce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731397729106 2024-11-12T07:48:52,572 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc68ebc818b4f18b462e5c4db154c48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731397729106 2024-11-12T07:48:52,572 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting cb98d636c65644bab8099f307b1041c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731397731283 2024-11-12T07:48:52,572 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98518ebb24c24f1d8fb762239775f03e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731397731276 2024-11-12T07:48:52,581 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:52,581 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/7d7331c2619c4b699e0abaf7364a5f83 is 50, key is test_row_0/B:col10/1731397731284/Put/seqid=0 2024-11-12T07:48:52,585 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:52,586 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/6e6ece0490ba4c0e8e15415e4897e53d is 50, key is test_row_0/A:col10/1731397731284/Put/seqid=0 2024-11-12T07:48:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742107_1283 (size=12663) 2024-11-12T07:48:52,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742108_1284 (size=12663) 2024-11-12T07:48:52,607 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/6e6ece0490ba4c0e8e15415e4897e53d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/6e6ece0490ba4c0e8e15415e4897e53d 2024-11-12T07:48:52,612 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/A of a03352d0297c5ba80936c4f6927633e5 into 6e6ece0490ba4c0e8e15415e4897e53d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:52,613 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:52,613 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/A, priority=13, startTime=1731397732570; duration=0sec 2024-11-12T07:48:52,613 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:52,613 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:A 2024-11-12T07:48:52,613 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:52,614 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:52,614 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/C is initiating minor compaction (all files) 2024-11-12T07:48:52,614 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/C in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,614 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/221d1922501c4ac49c6f7fc83e507436, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b3c18f749c784e248731c95cea6599c8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d95ccb79a3aa4c77ad7ef4a2581f27c4] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.0 K 2024-11-12T07:48:52,614 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221d1922501c4ac49c6f7fc83e507436, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731397727941 2024-11-12T07:48:52,614 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3c18f749c784e248731c95cea6599c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731397729106 2024-11-12T07:48:52,615 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d95ccb79a3aa4c77ad7ef4a2581f27c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731397731283 2024-11-12T07:48:52,622 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:52,622 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/d3261964ef91412a8397f889ac9141fc is 50, key is test_row_0/C:col10/1731397731284/Put/seqid=0 2024-11-12T07:48:52,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742109_1285 (size=12663) 2024-11-12T07:48:52,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-12T07:48:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:52,700 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T07:48:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:52,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:52,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e5ae99cc464647beb38574f75a985116 is 50, key is test_row_0/A:col10/1731397731339/Put/seqid=0 2024-11-12T07:48:52,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742110_1286 (size=12151) 2024-11-12T07:48:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T07:48:52,997 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/7d7331c2619c4b699e0abaf7364a5f83 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/7d7331c2619c4b699e0abaf7364a5f83 2024-11-12T07:48:53,002 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/B of a03352d0297c5ba80936c4f6927633e5 into 7d7331c2619c4b699e0abaf7364a5f83(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:53,002 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:53,002 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/B, priority=13, startTime=1731397732570; duration=0sec 2024-11-12T07:48:53,002 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:53,002 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:B 2024-11-12T07:48:53,032 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/d3261964ef91412a8397f889ac9141fc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d3261964ef91412a8397f889ac9141fc 2024-11-12T07:48:53,040 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/C of a03352d0297c5ba80936c4f6927633e5 into d3261964ef91412a8397f889ac9141fc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:53,040 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:53,040 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/C, priority=13, startTime=1731397732570; duration=0sec 2024-11-12T07:48:53,040 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:53,040 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:C 2024-11-12T07:48:53,109 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e5ae99cc464647beb38574f75a985116 2024-11-12T07:48:53,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/bdbc0b5ea684481ea6514cc7fd70d51f is 50, key is test_row_0/B:col10/1731397731339/Put/seqid=0 2024-11-12T07:48:53,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742111_1287 (size=12151) 2024-11-12T07:48:53,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:53,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:53,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397793519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397793520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,524 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/bdbc0b5ea684481ea6514cc7fd70d51f 2024-11-12T07:48:53,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397793521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b08f9a54ecec40098d17c5ba7e72406e is 50, key is test_row_0/C:col10/1731397731339/Put/seqid=0 2024-11-12T07:48:53,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742112_1288 (size=12151) 2024-11-12T07:48:53,536 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b08f9a54ecec40098d17c5ba7e72406e 2024-11-12T07:48:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e5ae99cc464647beb38574f75a985116 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e5ae99cc464647beb38574f75a985116 2024-11-12T07:48:53,549 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e5ae99cc464647beb38574f75a985116, entries=150, sequenceid=219, filesize=11.9 K 2024-11-12T07:48:53,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/bdbc0b5ea684481ea6514cc7fd70d51f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/bdbc0b5ea684481ea6514cc7fd70d51f 2024-11-12T07:48:53,555 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/bdbc0b5ea684481ea6514cc7fd70d51f, entries=150, sequenceid=219, filesize=11.9 K 2024-11-12T07:48:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b08f9a54ecec40098d17c5ba7e72406e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b08f9a54ecec40098d17c5ba7e72406e 2024-11-12T07:48:53,559 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b08f9a54ecec40098d17c5ba7e72406e, entries=150, sequenceid=219, filesize=11.9 K 2024-11-12T07:48:53,560 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a03352d0297c5ba80936c4f6927633e5 in 860ms, sequenceid=219, compaction requested=false 2024-11-12T07:48:53,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:53,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:53,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-12T07:48:53,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-12T07:48:53,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-12T07:48:53,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9310 sec 2024-11-12T07:48:53,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9360 sec 2024-11-12T07:48:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:53,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T07:48:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:53,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:53,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/9a39448899ab4275823043de23b921f6 is 50, key is test_row_0/A:col10/1731397733517/Put/seqid=0 2024-11-12T07:48:53,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742113_1289 (size=12151) 2024-11-12T07:48:53,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/9a39448899ab4275823043de23b921f6 2024-11-12T07:48:53,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a940be78960141bcb78451fa5326d68d is 50, key is test_row_0/B:col10/1731397733517/Put/seqid=0 2024-11-12T07:48:53,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742114_1290 (size=12151) 2024-11-12T07:48:53,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397793646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397793647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397793649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397793757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397793758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397793758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T07:48:53,778 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:53,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-12T07:48:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T07:48:53,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:53,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:53,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T07:48:53,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T07:48:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:53,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:53,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397793968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397793968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397793968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a940be78960141bcb78451fa5326d68d 2024-11-12T07:48:54,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/21b631e69b9044058edc0116df2d3ba8 is 50, key is test_row_0/C:col10/1731397733517/Put/seqid=0 2024-11-12T07:48:54,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742115_1291 (size=12151) 2024-11-12T07:48:54,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T07:48:54,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:54,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T07:48:54,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T07:48:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397794278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397794278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397794278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T07:48:54,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:54,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T07:48:54,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/21b631e69b9044058edc0116df2d3ba8 2024-11-12T07:48:54,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/9a39448899ab4275823043de23b921f6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/9a39448899ab4275823043de23b921f6 2024-11-12T07:48:54,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/9a39448899ab4275823043de23b921f6, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T07:48:54,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/a940be78960141bcb78451fa5326d68d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a940be78960141bcb78451fa5326d68d 2024-11-12T07:48:54,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a940be78960141bcb78451fa5326d68d, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T07:48:54,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/21b631e69b9044058edc0116df2d3ba8 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/21b631e69b9044058edc0116df2d3ba8 2024-11-12T07:48:54,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/21b631e69b9044058edc0116df2d3ba8, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T07:48:54,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a03352d0297c5ba80936c4f6927633e5 in 847ms, sequenceid=239, compaction requested=true 2024-11-12T07:48:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:54,477 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:54,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T07:48:54,477 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:54,478 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:54,478 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:54,479 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/B is initiating minor compaction (all files) 2024-11-12T07:48:54,479 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/A is initiating minor compaction (all files) 2024-11-12T07:48:54,479 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/A in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,479 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/B in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,479 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/7d7331c2619c4b699e0abaf7364a5f83, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/bdbc0b5ea684481ea6514cc7fd70d51f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a940be78960141bcb78451fa5326d68d] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.1 K 2024-11-12T07:48:54,479 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/6e6ece0490ba4c0e8e15415e4897e53d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e5ae99cc464647beb38574f75a985116, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/9a39448899ab4275823043de23b921f6] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.1 K 2024-11-12T07:48:54,479 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d7331c2619c4b699e0abaf7364a5f83, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731397731283 2024-11-12T07:48:54,479 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6ece0490ba4c0e8e15415e4897e53d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731397731283 2024-11-12T07:48:54,479 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting bdbc0b5ea684481ea6514cc7fd70d51f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731397731329 2024-11-12T07:48:54,479 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ae99cc464647beb38574f75a985116, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731397731329 2024-11-12T07:48:54,480 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting a940be78960141bcb78451fa5326d68d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731397733517 2024-11-12T07:48:54,480 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a39448899ab4275823043de23b921f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731397733517 2024-11-12T07:48:54,486 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:54,486 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:54,486 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/972834827ee4416d97fc67945628c321 is 50, key is test_row_0/A:col10/1731397733517/Put/seqid=0 2024-11-12T07:48:54,487 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/c12d988a6b3342cabfe3f0ec707d698f is 50, key is test_row_0/B:col10/1731397733517/Put/seqid=0 2024-11-12T07:48:54,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742116_1292 (size=12765) 2024-11-12T07:48:54,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742117_1293 (size=12765) 2024-11-12T07:48:54,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-12T07:48:54,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:48:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:54,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/a1634a0727cc4c2ababf51172724a74f is 50, key is test_row_0/A:col10/1731397733636/Put/seqid=0 2024-11-12T07:48:54,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742118_1294 (size=12151) 2024-11-12T07:48:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:54,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397794810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397794811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397794813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,895 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/c12d988a6b3342cabfe3f0ec707d698f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c12d988a6b3342cabfe3f0ec707d698f 2024-11-12T07:48:54,896 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/972834827ee4416d97fc67945628c321 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/972834827ee4416d97fc67945628c321 2024-11-12T07:48:54,899 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/B of a03352d0297c5ba80936c4f6927633e5 into c12d988a6b3342cabfe3f0ec707d698f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:54,899 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/A of a03352d0297c5ba80936c4f6927633e5 into 972834827ee4416d97fc67945628c321(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:54,899 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:54,899 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:54,899 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/B, priority=13, startTime=1731397734477; duration=0sec 2024-11-12T07:48:54,899 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/A, priority=13, startTime=1731397734477; duration=0sec 2024-11-12T07:48:54,900 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:54,900 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:A 2024-11-12T07:48:54,900 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:54,900 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:B 2024-11-12T07:48:54,900 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:54,901 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:54,901 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/C is initiating minor compaction (all files) 2024-11-12T07:48:54,901 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/C in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:54,901 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d3261964ef91412a8397f889ac9141fc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b08f9a54ecec40098d17c5ba7e72406e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/21b631e69b9044058edc0116df2d3ba8] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.1 K 2024-11-12T07:48:54,901 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3261964ef91412a8397f889ac9141fc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731397731283 2024-11-12T07:48:54,901 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting b08f9a54ecec40098d17c5ba7e72406e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731397731329 2024-11-12T07:48:54,902 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b631e69b9044058edc0116df2d3ba8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731397733517 2024-11-12T07:48:54,908 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:54,909 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/535b3ba6d59744439343c02e6ba557ab is 50, key is test_row_0/C:col10/1731397733517/Put/seqid=0 2024-11-12T07:48:54,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742119_1295 (size=12765) 2024-11-12T07:48:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T07:48:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397794918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397794918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397794927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:54,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:54,954 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/a1634a0727cc4c2ababf51172724a74f 2024-11-12T07:48:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/17e9b5fcc1dc47c782ddef0af7c10889 is 50, key is test_row_0/B:col10/1731397733636/Put/seqid=0 2024-11-12T07:48:54,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742120_1296 (size=12151) 2024-11-12T07:48:54,965 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/17e9b5fcc1dc47c782ddef0af7c10889 2024-11-12T07:48:54,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b4b9e961cea447e6b896d7fafa04618f is 50, key is test_row_0/C:col10/1731397733636/Put/seqid=0 2024-11-12T07:48:54,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742121_1297 (size=12151) 2024-11-12T07:48:55,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397795128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397795128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397795138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,317 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/535b3ba6d59744439343c02e6ba557ab as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/535b3ba6d59744439343c02e6ba557ab 2024-11-12T07:48:55,321 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/C of a03352d0297c5ba80936c4f6927633e5 into 535b3ba6d59744439343c02e6ba557ab(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:55,321 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:55,321 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/C, priority=13, startTime=1731397734477; duration=0sec 2024-11-12T07:48:55,321 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:55,321 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:C 2024-11-12T07:48:55,383 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b4b9e961cea447e6b896d7fafa04618f 2024-11-12T07:48:55,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/a1634a0727cc4c2ababf51172724a74f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/a1634a0727cc4c2ababf51172724a74f 2024-11-12T07:48:55,394 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/a1634a0727cc4c2ababf51172724a74f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-12T07:48:55,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/17e9b5fcc1dc47c782ddef0af7c10889 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/17e9b5fcc1dc47c782ddef0af7c10889 2024-11-12T07:48:55,398 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/17e9b5fcc1dc47c782ddef0af7c10889, entries=150, sequenceid=253, filesize=11.9 K 2024-11-12T07:48:55,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/b4b9e961cea447e6b896d7fafa04618f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b4b9e961cea447e6b896d7fafa04618f 2024-11-12T07:48:55,402 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b4b9e961cea447e6b896d7fafa04618f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-12T07:48:55,403 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a03352d0297c5ba80936c4f6927633e5 in 859ms, sequenceid=253, compaction requested=false 2024-11-12T07:48:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-12T07:48:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-12T07:48:55,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-12T07:48:55,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6230 sec 2024-11-12T07:48:55,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6270 sec 2024-11-12T07:48:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:55,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T07:48:55,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:55,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:55,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:55,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:55,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:55,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:55,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/16c56653ed4b496281006e1bcf8d858d is 50, key is test_row_0/A:col10/1731397735440/Put/seqid=0 2024-11-12T07:48:55,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742122_1298 (size=14741) 2024-11-12T07:48:55,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397795456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397795456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397795460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397795568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397795568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397795568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397795778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397795778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397795778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:55,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/16c56653ed4b496281006e1bcf8d858d 2024-11-12T07:48:55,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/c8ed4de09dd34125b46dd8ddc9306c3f is 50, key is test_row_0/B:col10/1731397735440/Put/seqid=0 2024-11-12T07:48:55,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742123_1299 (size=12301) 2024-11-12T07:48:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-12T07:48:55,928 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:55,929 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-12T07:48:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T07:48:55,930 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:55,931 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:55,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T07:48:56,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T07:48:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397796087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397796087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:56,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397796088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:56,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T07:48:56,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:56,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T07:48:56,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/c8ed4de09dd34125b46dd8ddc9306c3f 2024-11-12T07:48:56,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/edc190c8c6a04fac8260214d37ae3ac6 is 50, key is test_row_0/C:col10/1731397735440/Put/seqid=0 2024-11-12T07:48:56,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742124_1300 (size=12301) 2024-11-12T07:48:56,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T07:48:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T07:48:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T07:48:56,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397796597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397796598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:56,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397796598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:56,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:56,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:56,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/edc190c8c6a04fac8260214d37ae3ac6 2024-11-12T07:48:56,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/16c56653ed4b496281006e1bcf8d858d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/16c56653ed4b496281006e1bcf8d858d 2024-11-12T07:48:56,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/16c56653ed4b496281006e1bcf8d858d, entries=200, sequenceid=276, filesize=14.4 K 2024-11-12T07:48:56,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/c8ed4de09dd34125b46dd8ddc9306c3f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c8ed4de09dd34125b46dd8ddc9306c3f 2024-11-12T07:48:56,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c8ed4de09dd34125b46dd8ddc9306c3f, entries=150, sequenceid=276, filesize=12.0 K 2024-11-12T07:48:56,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/edc190c8c6a04fac8260214d37ae3ac6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/edc190c8c6a04fac8260214d37ae3ac6 2024-11-12T07:48:56,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/edc190c8c6a04fac8260214d37ae3ac6, entries=150, sequenceid=276, filesize=12.0 K 2024-11-12T07:48:56,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a03352d0297c5ba80936c4f6927633e5 in 1247ms, sequenceid=276, compaction requested=true 2024-11-12T07:48:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:56,687 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:56,687 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:56,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:56,688 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:56,688 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:56,688 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/A is initiating minor compaction (all files) 2024-11-12T07:48:56,688 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/B is initiating minor compaction (all files) 2024-11-12T07:48:56,688 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/A in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,688 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/B in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,688 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/972834827ee4416d97fc67945628c321, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/a1634a0727cc4c2ababf51172724a74f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/16c56653ed4b496281006e1bcf8d858d] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=38.7 K 2024-11-12T07:48:56,688 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c12d988a6b3342cabfe3f0ec707d698f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/17e9b5fcc1dc47c782ddef0af7c10889, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c8ed4de09dd34125b46dd8ddc9306c3f] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.3 K 2024-11-12T07:48:56,688 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c12d988a6b3342cabfe3f0ec707d698f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731397733517 2024-11-12T07:48:56,688 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 972834827ee4416d97fc67945628c321, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731397733517 2024-11-12T07:48:56,689 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 17e9b5fcc1dc47c782ddef0af7c10889, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731397733636 2024-11-12T07:48:56,689 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1634a0727cc4c2ababf51172724a74f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731397733636 2024-11-12T07:48:56,689 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ed4de09dd34125b46dd8ddc9306c3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731397734800 2024-11-12T07:48:56,689 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16c56653ed4b496281006e1bcf8d858d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731397734800 2024-11-12T07:48:56,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-12T07:48:56,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:56,695 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:48:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:56,696 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:56,697 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:56,697 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/8848eaa920bb4486a175674506816989 is 50, key is test_row_0/B:col10/1731397735440/Put/seqid=0 2024-11-12T07:48:56,697 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/2b815fc407c148b78de15ea91df35786 is 50, key is test_row_0/A:col10/1731397735440/Put/seqid=0 2024-11-12T07:48:56,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/d48175310f7f4c0ea8e78bd28223969e is 50, key is test_row_0/A:col10/1731397735455/Put/seqid=0 2024-11-12T07:48:56,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742125_1301 (size=13017) 2024-11-12T07:48:56,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742126_1302 (size=13017) 2024-11-12T07:48:56,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742127_1303 (size=12301) 2024-11-12T07:48:56,725 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/d48175310f7f4c0ea8e78bd28223969e 2024-11-12T07:48:56,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/90207d9130794ea69adca85879cab5b3 is 50, key is test_row_0/B:col10/1731397735455/Put/seqid=0 2024-11-12T07:48:56,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742128_1304 (size=12301) 2024-11-12T07:48:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T07:48:57,115 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/8848eaa920bb4486a175674506816989 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/8848eaa920bb4486a175674506816989 2024-11-12T07:48:57,119 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/B of a03352d0297c5ba80936c4f6927633e5 into 8848eaa920bb4486a175674506816989(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:57,119 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:57,119 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/B, priority=13, startTime=1731397736687; duration=0sec 2024-11-12T07:48:57,119 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:57,119 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:B 2024-11-12T07:48:57,119 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:57,120 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:57,120 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/C is initiating minor compaction (all files) 2024-11-12T07:48:57,120 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/C in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:57,120 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/535b3ba6d59744439343c02e6ba557ab, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b4b9e961cea447e6b896d7fafa04618f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/edc190c8c6a04fac8260214d37ae3ac6] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.3 K 2024-11-12T07:48:57,120 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 535b3ba6d59744439343c02e6ba557ab, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731397733517 2024-11-12T07:48:57,121 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b9e961cea447e6b896d7fafa04618f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731397733636 2024-11-12T07:48:57,121 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting edc190c8c6a04fac8260214d37ae3ac6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731397734800 2024-11-12T07:48:57,123 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/2b815fc407c148b78de15ea91df35786 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/2b815fc407c148b78de15ea91df35786 2024-11-12T07:48:57,127 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:57,128 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/A of a03352d0297c5ba80936c4f6927633e5 into 2b815fc407c148b78de15ea91df35786(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:57,128 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:57,128 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/A, priority=13, startTime=1731397736687; duration=0sec 2024-11-12T07:48:57,128 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/69c69133442f4c5b9d0701faf24e6056 is 50, key is test_row_0/C:col10/1731397735440/Put/seqid=0 2024-11-12T07:48:57,128 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:57,128 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:A 2024-11-12T07:48:57,135 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/90207d9130794ea69adca85879cab5b3 2024-11-12T07:48:57,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742129_1305 (size=13017) 2024-11-12T07:48:57,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/aa452957588e4efea8b938ed949c715f is 50, key is test_row_0/C:col10/1731397735455/Put/seqid=0 2024-11-12T07:48:57,146 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/69c69133442f4c5b9d0701faf24e6056 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69c69133442f4c5b9d0701faf24e6056 2024-11-12T07:48:57,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742130_1306 (size=12301) 2024-11-12T07:48:57,151 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/aa452957588e4efea8b938ed949c715f 2024-11-12T07:48:57,151 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/C of a03352d0297c5ba80936c4f6927633e5 into 69c69133442f4c5b9d0701faf24e6056(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:57,151 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:57,151 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/C, priority=13, startTime=1731397736687; duration=0sec 2024-11-12T07:48:57,151 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:57,151 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:C 2024-11-12T07:48:57,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/d48175310f7f4c0ea8e78bd28223969e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/d48175310f7f4c0ea8e78bd28223969e 2024-11-12T07:48:57,158 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/d48175310f7f4c0ea8e78bd28223969e, entries=150, sequenceid=288, filesize=12.0 K 2024-11-12T07:48:57,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/90207d9130794ea69adca85879cab5b3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/90207d9130794ea69adca85879cab5b3 2024-11-12T07:48:57,163 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/90207d9130794ea69adca85879cab5b3, entries=150, sequenceid=288, filesize=12.0 K 2024-11-12T07:48:57,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/aa452957588e4efea8b938ed949c715f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/aa452957588e4efea8b938ed949c715f 2024-11-12T07:48:57,168 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/aa452957588e4efea8b938ed949c715f, entries=150, sequenceid=288, filesize=12.0 K 2024-11-12T07:48:57,169 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for a03352d0297c5ba80936c4f6927633e5 in 474ms, sequenceid=288, compaction requested=false 2024-11-12T07:48:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-12T07:48:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-12T07:48:57,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-12T07:48:57,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2410 sec 2024-11-12T07:48:57,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.2450 sec 2024-11-12T07:48:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:57,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:48:57,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:57,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:57,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:57,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:57,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:57,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:48:57,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/da91b6cbcb984e10a07737bf0982482e is 50, key is test_row_0/A:col10/1731397737531/Put/seqid=0 2024-11-12T07:48:57,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742131_1307 (size=12301) 2024-11-12T07:48:57,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397737598, max timestamp=1731397737607Num uniques 859; 2024-11-12T07:48:57,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:48:57,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397737598, max timestamp=1731397737607Num uniques 859; 2024-11-12T07:48:57,612 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:48:57,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731397737598, max timestamp=1731397737607Num uniques 859; 2024-11-12T07:48:57,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T07:48:57,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397797625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397797624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397797655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397797655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397797655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397797758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397797758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397797768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397797768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397797768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/da91b6cbcb984e10a07737bf0982482e 2024-11-12T07:48:57,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/028326c08fff4cafb3b4a95c0f0649aa is 50, key is test_row_0/B:col10/1731397737531/Put/seqid=0 2024-11-12T07:48:57,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742132_1308 (size=12301) 2024-11-12T07:48:57,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397797968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397797968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397797977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397797977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397797977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:58,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T07:48:58,078 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:48:58,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:48:58,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-12T07:48:58,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T07:48:58,092 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:48:58,093 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:48:58,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:48:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T07:48:58,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397798278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397798278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:58,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50904 deadline: 1731397798287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:58,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1731397798287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:58,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50944 deadline: 1731397798288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:58,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/028326c08fff4cafb3b4a95c0f0649aa 2024-11-12T07:48:58,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/3049fb11f62f4e67a619caaa7854c051 is 50, key is test_row_0/C:col10/1731397737531/Put/seqid=0 2024-11-12T07:48:58,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742133_1309 (size=12301) 2024-11-12T07:48:58,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T07:48:58,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:58,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:58,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T07:48:58,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/3049fb11f62f4e67a619caaa7854c051 2024-11-12T07:48:58,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/da91b6cbcb984e10a07737bf0982482e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/da91b6cbcb984e10a07737bf0982482e 2024-11-12T07:48:58,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/da91b6cbcb984e10a07737bf0982482e, entries=150, sequenceid=301, filesize=12.0 K 2024-11-12T07:48:58,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/028326c08fff4cafb3b4a95c0f0649aa as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/028326c08fff4cafb3b4a95c0f0649aa 2024-11-12T07:48:58,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/028326c08fff4cafb3b4a95c0f0649aa, entries=150, sequenceid=301, filesize=12.0 K 2024-11-12T07:48:58,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50916 deadline: 1731397798788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:58,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/3049fb11f62f4e67a619caaa7854c051 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/3049fb11f62f4e67a619caaa7854c051 2024-11-12T07:48:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:48:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1731397798788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:48:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:48:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:48:58,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/3049fb11f62f4e67a619caaa7854c051, entries=150, sequenceid=301, filesize=12.0 K 2024-11-12T07:48:58,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for a03352d0297c5ba80936c4f6927633e5 in 1256ms, sequenceid=301, compaction requested=true 2024-11-12T07:48:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:48:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:58,795 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:48:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:58,795 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:58,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a03352d0297c5ba80936c4f6927633e5:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:48:58,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:58,796 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:58,796 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:58,796 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/B is initiating minor compaction (all files) 2024-11-12T07:48:58,796 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/A is initiating minor compaction (all files) 2024-11-12T07:48:58,796 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/A in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,796 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/B in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,796 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/8848eaa920bb4486a175674506816989, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/90207d9130794ea69adca85879cab5b3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/028326c08fff4cafb3b4a95c0f0649aa] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.7 K 2024-11-12T07:48:58,796 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/2b815fc407c148b78de15ea91df35786, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/d48175310f7f4c0ea8e78bd28223969e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/da91b6cbcb984e10a07737bf0982482e] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.7 K 2024-11-12T07:48:58,797 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b815fc407c148b78de15ea91df35786, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731397734800 2024-11-12T07:48:58,797 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8848eaa920bb4486a175674506816989, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731397734800 2024-11-12T07:48:58,797 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48175310f7f4c0ea8e78bd28223969e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731397735444 2024-11-12T07:48:58,797 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting da91b6cbcb984e10a07737bf0982482e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731397737531 2024-11-12T07:48:58,797 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 90207d9130794ea69adca85879cab5b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731397735444 2024-11-12T07:48:58,798 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 028326c08fff4cafb3b4a95c0f0649aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731397737531 2024-11-12T07:48:58,803 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:58,803 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/01e867d1de7845cd9086d969ad822042 is 50, key is test_row_0/A:col10/1731397737531/Put/seqid=0 2024-11-12T07:48:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:48:58,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=140.49 KB heapSize=404.70 KB 2024-11-12T07:48:58,805 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-12T07:48:58,805 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:58,805 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,805 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-12T07:48:58,805 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:58,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:48:58,805 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:48:58,806 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:48:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:48:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:48:58,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:48:58,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:58,806 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,805 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-11-12T07:48:58,806 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:58,806 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,806 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:58,809 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:58,809 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/18136738f2a342c386843aed55e884a0 is 50, key is test_row_0/B:col10/1731397737531/Put/seqid=0 2024-11-12T07:48:58,810 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-12T07:48:58,811 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:58,811 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:58,811 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,811 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-11-12T07:48:58,811 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:58,811 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,811 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:58,812 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-12T07:48:58,812 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:58,812 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,812 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:58,812 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-12T07:48:58,812 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:58,812 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,812 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:58,813 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-12T07:48:58,813 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:58,813 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,813 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:58,813 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:58,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742134_1310 (size=13119) 2024-11-12T07:48:58,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742135_1311 (size=13119) 2024-11-12T07:48:58,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7cdde9ccda6d4f70b305cf67c20313b8 is 50, key is test_row_0/A:col10/1731397737625/Put/seqid=0 2024-11-12T07:48:58,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742136_1312 (size=14741) 2024-11-12T07:48:58,819 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/18136738f2a342c386843aed55e884a0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/18136738f2a342c386843aed55e884a0 2024-11-12T07:48:58,822 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/B of a03352d0297c5ba80936c4f6927633e5 into 18136738f2a342c386843aed55e884a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:58,822 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:58,822 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/B, priority=13, startTime=1731397738795; duration=0sec 2024-11-12T07:48:58,823 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:48:58,823 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:B 2024-11-12T07:48:58,823 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:48:58,823 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:48:58,823 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): a03352d0297c5ba80936c4f6927633e5/C is initiating minor compaction (all files) 2024-11-12T07:48:58,824 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a03352d0297c5ba80936c4f6927633e5/C in TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,824 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69c69133442f4c5b9d0701faf24e6056, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/aa452957588e4efea8b938ed949c715f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/3049fb11f62f4e67a619caaa7854c051] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp, totalSize=36.7 K 2024-11-12T07:48:58,824 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c69133442f4c5b9d0701faf24e6056, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731397734800 2024-11-12T07:48:58,824 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting aa452957588e4efea8b938ed949c715f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731397735444 2024-11-12T07:48:58,824 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 3049fb11f62f4e67a619caaa7854c051, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731397737531 2024-11-12T07:48:58,829 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a03352d0297c5ba80936c4f6927633e5#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:48:58,830 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/5127c39cd7564987ae3679ba0e98c8f3 is 50, key is test_row_0/C:col10/1731397737531/Put/seqid=0 2024-11-12T07:48:58,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742137_1313 (size=13119) 2024-11-12T07:48:58,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:58,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,129 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:48:59,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:59,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:59,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7cdde9ccda6d4f70b305cf67c20313b8 2024-11-12T07:48:59,223 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/01e867d1de7845cd9086d969ad822042 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/01e867d1de7845cd9086d969ad822042 2024-11-12T07:48:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T07:48:59,229 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/A of a03352d0297c5ba80936c4f6927633e5 into 01e867d1de7845cd9086d969ad822042(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:59,229 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:59,229 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/A, priority=13, startTime=1731397738795; duration=0sec 2024-11-12T07:48:59,229 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:59,229 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:A 2024-11-12T07:48:59,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/420db2ab17794e299d01d1a7d4a7e916 is 50, key is test_row_0/B:col10/1731397737625/Put/seqid=0 2024-11-12T07:48:59,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742138_1314 (size=12301) 2024-11-12T07:48:59,237 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/5127c39cd7564987ae3679ba0e98c8f3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/5127c39cd7564987ae3679ba0e98c8f3 2024-11-12T07:48:59,241 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a03352d0297c5ba80936c4f6927633e5/C of a03352d0297c5ba80936c4f6927633e5 into 5127c39cd7564987ae3679ba0e98c8f3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:48:59,241 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:48:59,241 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5., storeName=a03352d0297c5ba80936c4f6927633e5/C, priority=13, startTime=1731397738795; duration=0sec 2024-11-12T07:48:59,241 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:48:59,241 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a03352d0297c5ba80936c4f6927633e5:C 2024-11-12T07:48:59,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/420db2ab17794e299d01d1a7d4a7e916 2024-11-12T07:48:59,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/92086a82ee7c42d7a4eb8f7e63afa06a is 50, key is test_row_0/C:col10/1731397737625/Put/seqid=0 2024-11-12T07:48:59,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742139_1315 (size=12301) 2024-11-12T07:48:59,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:59,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:59,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,803 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-12T07:48:59,803 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:59,803 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-12T07:48:59,803 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:59,803 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:48:59,803 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:59,803 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:59,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:59,803 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:48:59,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:48:59,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:48:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. as already flushing 2024-11-12T07:48:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:48:59,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:48:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:00,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.04 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/92086a82ee7c42d7a4eb8f7e63afa06a 2024-11-12T07:49:00,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/7cdde9ccda6d4f70b305cf67c20313b8 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7cdde9ccda6d4f70b305cf67c20313b8 2024-11-12T07:49:00,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7cdde9ccda6d4f70b305cf67c20313b8, entries=200, sequenceid=341, filesize=14.4 K 2024-11-12T07:49:00,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/420db2ab17794e299d01d1a7d4a7e916 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/420db2ab17794e299d01d1a7d4a7e916 2024-11-12T07:49:00,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/420db2ab17794e299d01d1a7d4a7e916, entries=150, sequenceid=341, filesize=12.0 K 2024-11-12T07:49:00,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/92086a82ee7c42d7a4eb8f7e63afa06a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/92086a82ee7c42d7a4eb8f7e63afa06a 2024-11-12T07:49:00,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/92086a82ee7c42d7a4eb8f7e63afa06a, entries=150, sequenceid=341, filesize=12.0 K 2024-11-12T07:49:00,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~153.91 KB/157605, heapSize ~444.49 KB/455160, currentSize=20.13 KB/20610 for a03352d0297c5ba80936c4f6927633e5 in 1268ms, sequenceid=341, compaction requested=false 2024-11-12T07:49:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:49:00,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-12T07:49:00,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:49:00,099 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing a03352d0297c5ba80936c4f6927633e5 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T07:49:00,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=A 2024-11-12T07:49:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=B 2024-11-12T07:49:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a03352d0297c5ba80936c4f6927633e5, store=C 2024-11-12T07:49:00,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:00,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e101dea3aa934cc2b53045c0d937f3c3 is 50, key is test_row_1/A:col10/1731397739800/Put/seqid=0 2024-11-12T07:49:00,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742140_1316 (size=7415) 2024-11-12T07:49:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T07:49:00,513 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e101dea3aa934cc2b53045c0d937f3c3 2024-11-12T07:49:00,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/78d1f920fb1e4da9bdc57830a07bc9d2 is 50, key is test_row_1/B:col10/1731397739800/Put/seqid=0 2024-11-12T07:49:00,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742141_1317 (size=7415) 2024-11-12T07:49:00,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/78d1f920fb1e4da9bdc57830a07bc9d2 2024-11-12T07:49:00,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/13ed0d87a7ea4ebca5a52d87e4023997 is 50, key is test_row_1/C:col10/1731397739800/Put/seqid=0 2024-11-12T07:49:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742142_1318 (size=7415) 2024-11-12T07:49:01,341 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/13ed0d87a7ea4ebca5a52d87e4023997 2024-11-12T07:49:01,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/A/e101dea3aa934cc2b53045c0d937f3c3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e101dea3aa934cc2b53045c0d937f3c3 2024-11-12T07:49:01,351 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e101dea3aa934cc2b53045c0d937f3c3, entries=50, sequenceid=350, filesize=7.2 K 2024-11-12T07:49:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/B/78d1f920fb1e4da9bdc57830a07bc9d2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/78d1f920fb1e4da9bdc57830a07bc9d2 2024-11-12T07:49:01,355 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/78d1f920fb1e4da9bdc57830a07bc9d2, entries=50, sequenceid=350, filesize=7.2 K 2024-11-12T07:49:01,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/.tmp/C/13ed0d87a7ea4ebca5a52d87e4023997 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/13ed0d87a7ea4ebca5a52d87e4023997 2024-11-12T07:49:01,358 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/13ed0d87a7ea4ebca5a52d87e4023997, entries=50, sequenceid=350, filesize=7.2 K 2024-11-12T07:49:01,359 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a03352d0297c5ba80936c4f6927633e5 in 1260ms, sequenceid=350, compaction requested=true 2024-11-12T07:49:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for a03352d0297c5ba80936c4f6927633e5: 2024-11-12T07:49:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:49:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-12T07:49:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-12T07:49:01,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-12T07:49:01,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2670 sec 2024-11-12T07:49:01,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 3.2840 sec 2024-11-12T07:49:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-12T07:49:02,258 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3664 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3613 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3721 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3642 2024-11-12T07:49:02,258 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3615 2024-11-12T07:49:02,259 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T07:49:02,259 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:49:02,259 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:49:02,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:02,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:02,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:02,260 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T07:49:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T07:49:02,264 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397742263"}]},"ts":"1731397742263"} 2024-11-12T07:49:02,265 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T07:49:02,265 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T07:49:02,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:49:02,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, UNASSIGN}] 2024-11-12T07:49:02,269 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, UNASSIGN 2024-11-12T07:49:02,270 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a03352d0297c5ba80936c4f6927633e5, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:02,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, UNASSIGN because future has completed 2024-11-12T07:49:02,271 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:49:02,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:49:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T07:49:02,426 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:49:02,426 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:49:02,426 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing a03352d0297c5ba80936c4f6927633e5, disabling compactions & flushes 2024-11-12T07:49:02,426 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:49:02,426 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:49:02,426 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. after waiting 0 ms 2024-11-12T07:49:02,427 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:49:02,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e84bfc879bd64ee6a2d07c2ae1f960fc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/755ea9b94f5746b796aef480acee6e19, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/56e8f109773046f0be5d9ee0faa625b8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/516e51f4557c4e8a9ce90dd56aa4c311, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7e94a43f37b940938aabc1f2afd14fd9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/4bbb2672f00c4b31ac6ed662495c509e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7848afc337124f799ec64ff56a808719, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/818010a0baa848b29df059e867cd1c32, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/80be6d4c7dab473da7691dc2cadd8050, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/53cb06ae67744b6b82ca0e91409d1a7a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/79f04a75b14a4c8c89ccd1e4670a7aa4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7f83aef0c8ad4c669654e63963ca27c5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/69b3d53c40914049b35abd5926808a16, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/3dc68ebc818b4f18b462e5c4db154c48, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/98518ebb24c24f1d8fb762239775f03e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/6e6ece0490ba4c0e8e15415e4897e53d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e5ae99cc464647beb38574f75a985116, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/972834827ee4416d97fc67945628c321, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/9a39448899ab4275823043de23b921f6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/a1634a0727cc4c2ababf51172724a74f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/16c56653ed4b496281006e1bcf8d858d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/2b815fc407c148b78de15ea91df35786, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/d48175310f7f4c0ea8e78bd28223969e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/da91b6cbcb984e10a07737bf0982482e] to archive 2024-11-12T07:49:02,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:49:02,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e84bfc879bd64ee6a2d07c2ae1f960fc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e84bfc879bd64ee6a2d07c2ae1f960fc 2024-11-12T07:49:02,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/755ea9b94f5746b796aef480acee6e19 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/755ea9b94f5746b796aef480acee6e19 2024-11-12T07:49:02,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/56e8f109773046f0be5d9ee0faa625b8 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/56e8f109773046f0be5d9ee0faa625b8 2024-11-12T07:49:02,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/516e51f4557c4e8a9ce90dd56aa4c311 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/516e51f4557c4e8a9ce90dd56aa4c311 2024-11-12T07:49:02,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7e94a43f37b940938aabc1f2afd14fd9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7e94a43f37b940938aabc1f2afd14fd9 2024-11-12T07:49:02,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/4bbb2672f00c4b31ac6ed662495c509e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/4bbb2672f00c4b31ac6ed662495c509e 2024-11-12T07:49:02,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7848afc337124f799ec64ff56a808719 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7848afc337124f799ec64ff56a808719 2024-11-12T07:49:02,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/818010a0baa848b29df059e867cd1c32 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/818010a0baa848b29df059e867cd1c32 2024-11-12T07:49:02,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/80be6d4c7dab473da7691dc2cadd8050 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/80be6d4c7dab473da7691dc2cadd8050 2024-11-12T07:49:02,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/53cb06ae67744b6b82ca0e91409d1a7a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/53cb06ae67744b6b82ca0e91409d1a7a 2024-11-12T07:49:02,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/79f04a75b14a4c8c89ccd1e4670a7aa4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/79f04a75b14a4c8c89ccd1e4670a7aa4 2024-11-12T07:49:02,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7f83aef0c8ad4c669654e63963ca27c5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7f83aef0c8ad4c669654e63963ca27c5 2024-11-12T07:49:02,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/69b3d53c40914049b35abd5926808a16 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/69b3d53c40914049b35abd5926808a16 2024-11-12T07:49:02,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/3dc68ebc818b4f18b462e5c4db154c48 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/3dc68ebc818b4f18b462e5c4db154c48 2024-11-12T07:49:02,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/98518ebb24c24f1d8fb762239775f03e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/98518ebb24c24f1d8fb762239775f03e 2024-11-12T07:49:02,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/6e6ece0490ba4c0e8e15415e4897e53d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/6e6ece0490ba4c0e8e15415e4897e53d 2024-11-12T07:49:02,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e5ae99cc464647beb38574f75a985116 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e5ae99cc464647beb38574f75a985116 2024-11-12T07:49:02,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/972834827ee4416d97fc67945628c321 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/972834827ee4416d97fc67945628c321 2024-11-12T07:49:02,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/9a39448899ab4275823043de23b921f6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/9a39448899ab4275823043de23b921f6 2024-11-12T07:49:02,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/a1634a0727cc4c2ababf51172724a74f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/a1634a0727cc4c2ababf51172724a74f 2024-11-12T07:49:02,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/16c56653ed4b496281006e1bcf8d858d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/16c56653ed4b496281006e1bcf8d858d 2024-11-12T07:49:02,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/2b815fc407c148b78de15ea91df35786 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/2b815fc407c148b78de15ea91df35786 2024-11-12T07:49:02,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/d48175310f7f4c0ea8e78bd28223969e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/d48175310f7f4c0ea8e78bd28223969e 2024-11-12T07:49:02,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/da91b6cbcb984e10a07737bf0982482e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/da91b6cbcb984e10a07737bf0982482e 2024-11-12T07:49:02,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/22c4151c003847068a5d69a59caaecd7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/d76e6adb5e4a464c89eac6a7cc5c7460, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/b1808687e04846d99473b31020af69cf, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/3825f9fd215d417ab861ca57c09119f6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/173ae990a67a48018b7c0e05b8f051cc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c5da6055396546059c8603d4f82828ad, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a50d89c502af4e88a27984ffaf041623, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/83dc9860272d4fa8bdef595379a1b4f2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/92fe703125f04ea6982ae6a6b0193a71, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/099aa36bf6524574b059475c0d0266d6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/0b9a823bdeab452bb089959f54506a96, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/351c3b39c75843bab0bf9bfad01d4da9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a61c68fbd10d4a9799a0ec7fab84b929, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/e37872be8c5247a088f7100b27a0dbce, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/7d7331c2619c4b699e0abaf7364a5f83, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/cb98d636c65644bab8099f307b1041c0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/bdbc0b5ea684481ea6514cc7fd70d51f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c12d988a6b3342cabfe3f0ec707d698f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a940be78960141bcb78451fa5326d68d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/17e9b5fcc1dc47c782ddef0af7c10889, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/8848eaa920bb4486a175674506816989, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c8ed4de09dd34125b46dd8ddc9306c3f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/90207d9130794ea69adca85879cab5b3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/028326c08fff4cafb3b4a95c0f0649aa] to archive 2024-11-12T07:49:02,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:49:02,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/22c4151c003847068a5d69a59caaecd7 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/22c4151c003847068a5d69a59caaecd7 2024-11-12T07:49:02,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/d76e6adb5e4a464c89eac6a7cc5c7460 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/d76e6adb5e4a464c89eac6a7cc5c7460 2024-11-12T07:49:02,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/b1808687e04846d99473b31020af69cf to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/b1808687e04846d99473b31020af69cf 2024-11-12T07:49:02,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/3825f9fd215d417ab861ca57c09119f6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/3825f9fd215d417ab861ca57c09119f6 2024-11-12T07:49:02,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/173ae990a67a48018b7c0e05b8f051cc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/173ae990a67a48018b7c0e05b8f051cc 2024-11-12T07:49:02,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c5da6055396546059c8603d4f82828ad to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c5da6055396546059c8603d4f82828ad 2024-11-12T07:49:02,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a50d89c502af4e88a27984ffaf041623 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a50d89c502af4e88a27984ffaf041623 2024-11-12T07:49:02,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/83dc9860272d4fa8bdef595379a1b4f2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/83dc9860272d4fa8bdef595379a1b4f2 2024-11-12T07:49:02,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/92fe703125f04ea6982ae6a6b0193a71 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/92fe703125f04ea6982ae6a6b0193a71 2024-11-12T07:49:02,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/099aa36bf6524574b059475c0d0266d6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/099aa36bf6524574b059475c0d0266d6 2024-11-12T07:49:02,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/0b9a823bdeab452bb089959f54506a96 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/0b9a823bdeab452bb089959f54506a96 2024-11-12T07:49:02,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/351c3b39c75843bab0bf9bfad01d4da9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/351c3b39c75843bab0bf9bfad01d4da9 2024-11-12T07:49:02,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a61c68fbd10d4a9799a0ec7fab84b929 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a61c68fbd10d4a9799a0ec7fab84b929 2024-11-12T07:49:02,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/e37872be8c5247a088f7100b27a0dbce to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/e37872be8c5247a088f7100b27a0dbce 2024-11-12T07:49:02,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/7d7331c2619c4b699e0abaf7364a5f83 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/7d7331c2619c4b699e0abaf7364a5f83 2024-11-12T07:49:02,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/cb98d636c65644bab8099f307b1041c0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/cb98d636c65644bab8099f307b1041c0 2024-11-12T07:49:02,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/bdbc0b5ea684481ea6514cc7fd70d51f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/bdbc0b5ea684481ea6514cc7fd70d51f 2024-11-12T07:49:02,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c12d988a6b3342cabfe3f0ec707d698f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c12d988a6b3342cabfe3f0ec707d698f 2024-11-12T07:49:02,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a940be78960141bcb78451fa5326d68d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/a940be78960141bcb78451fa5326d68d 2024-11-12T07:49:02,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/17e9b5fcc1dc47c782ddef0af7c10889 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/17e9b5fcc1dc47c782ddef0af7c10889 2024-11-12T07:49:02,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/8848eaa920bb4486a175674506816989 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/8848eaa920bb4486a175674506816989 2024-11-12T07:49:02,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c8ed4de09dd34125b46dd8ddc9306c3f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/c8ed4de09dd34125b46dd8ddc9306c3f 2024-11-12T07:49:02,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/90207d9130794ea69adca85879cab5b3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/90207d9130794ea69adca85879cab5b3 2024-11-12T07:49:02,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/028326c08fff4cafb3b4a95c0f0649aa to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/028326c08fff4cafb3b4a95c0f0649aa 2024-11-12T07:49:02,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/039daa3b35f442ec857839427342ed89, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/9288ff7d568049f0bd4526468b6c6f79, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/06162777cd9a4cad9c6c3a0410ef4c26, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/dc51a03a7c2e46048c3f1e9f51a1a816, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/99abf850640d45178c0fe0d9ed920669, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/5210b48b9d744ea28108199402a456b7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/199516896373432c998cf0faa1ad651d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/403a30631b99471c80077533777a510e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69556b14be564885b5609bb21db450ff, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/e20ce25f2d3346b0825364c9ab4a89e3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/f6ef1588b3dd4d63b8d9414810f3c280, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/221d1922501c4ac49c6f7fc83e507436, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/2a18a622a93f4bc78e7cd624ca2eda4e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b3c18f749c784e248731c95cea6599c8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d3261964ef91412a8397f889ac9141fc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d95ccb79a3aa4c77ad7ef4a2581f27c4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b08f9a54ecec40098d17c5ba7e72406e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/535b3ba6d59744439343c02e6ba557ab, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/21b631e69b9044058edc0116df2d3ba8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b4b9e961cea447e6b896d7fafa04618f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69c69133442f4c5b9d0701faf24e6056, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/edc190c8c6a04fac8260214d37ae3ac6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/aa452957588e4efea8b938ed949c715f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/3049fb11f62f4e67a619caaa7854c051] to archive 2024-11-12T07:49:02,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:49:02,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/039daa3b35f442ec857839427342ed89 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/039daa3b35f442ec857839427342ed89 2024-11-12T07:49:02,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/9288ff7d568049f0bd4526468b6c6f79 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/9288ff7d568049f0bd4526468b6c6f79 2024-11-12T07:49:02,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/06162777cd9a4cad9c6c3a0410ef4c26 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/06162777cd9a4cad9c6c3a0410ef4c26 2024-11-12T07:49:02,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/dc51a03a7c2e46048c3f1e9f51a1a816 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/dc51a03a7c2e46048c3f1e9f51a1a816 2024-11-12T07:49:02,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/99abf850640d45178c0fe0d9ed920669 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/99abf850640d45178c0fe0d9ed920669 2024-11-12T07:49:02,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/5210b48b9d744ea28108199402a456b7 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/5210b48b9d744ea28108199402a456b7 2024-11-12T07:49:02,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/199516896373432c998cf0faa1ad651d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/199516896373432c998cf0faa1ad651d 2024-11-12T07:49:02,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/403a30631b99471c80077533777a510e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/403a30631b99471c80077533777a510e 2024-11-12T07:49:02,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69556b14be564885b5609bb21db450ff to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69556b14be564885b5609bb21db450ff 2024-11-12T07:49:02,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/e20ce25f2d3346b0825364c9ab4a89e3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/e20ce25f2d3346b0825364c9ab4a89e3 2024-11-12T07:49:02,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/f6ef1588b3dd4d63b8d9414810f3c280 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/f6ef1588b3dd4d63b8d9414810f3c280 2024-11-12T07:49:02,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/221d1922501c4ac49c6f7fc83e507436 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/221d1922501c4ac49c6f7fc83e507436 2024-11-12T07:49:02,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/2a18a622a93f4bc78e7cd624ca2eda4e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/2a18a622a93f4bc78e7cd624ca2eda4e 2024-11-12T07:49:02,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b3c18f749c784e248731c95cea6599c8 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b3c18f749c784e248731c95cea6599c8 2024-11-12T07:49:02,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d3261964ef91412a8397f889ac9141fc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d3261964ef91412a8397f889ac9141fc 2024-11-12T07:49:02,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d95ccb79a3aa4c77ad7ef4a2581f27c4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/d95ccb79a3aa4c77ad7ef4a2581f27c4 2024-11-12T07:49:02,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b08f9a54ecec40098d17c5ba7e72406e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b08f9a54ecec40098d17c5ba7e72406e 2024-11-12T07:49:02,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/535b3ba6d59744439343c02e6ba557ab to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/535b3ba6d59744439343c02e6ba557ab 2024-11-12T07:49:02,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/21b631e69b9044058edc0116df2d3ba8 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/21b631e69b9044058edc0116df2d3ba8 2024-11-12T07:49:02,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b4b9e961cea447e6b896d7fafa04618f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/b4b9e961cea447e6b896d7fafa04618f 2024-11-12T07:49:02,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69c69133442f4c5b9d0701faf24e6056 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/69c69133442f4c5b9d0701faf24e6056 2024-11-12T07:49:02,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/edc190c8c6a04fac8260214d37ae3ac6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/edc190c8c6a04fac8260214d37ae3ac6 2024-11-12T07:49:02,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/aa452957588e4efea8b938ed949c715f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/aa452957588e4efea8b938ed949c715f 2024-11-12T07:49:02,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/3049fb11f62f4e67a619caaa7854c051 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/3049fb11f62f4e67a619caaa7854c051 2024-11-12T07:49:02,502 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/recovered.edits/353.seqid, newMaxSeqId=353, maxSeqId=1 2024-11-12T07:49:02,502 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5. 2024-11-12T07:49:02,503 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for a03352d0297c5ba80936c4f6927633e5: Waiting for close lock at 1731397742426Running coprocessor pre-close hooks at 1731397742426Disabling compacts and flushes for region at 1731397742426Disabling writes for close at 1731397742426Writing region close event to WAL at 1731397742500 (+74 ms)Running coprocessor post-close hooks at 1731397742502 (+2 ms)Closed at 1731397742502 2024-11-12T07:49:02,504 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:49:02,504 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a03352d0297c5ba80936c4f6927633e5, regionState=CLOSED 2024-11-12T07:49:02,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:49:02,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-12T07:49:02,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure a03352d0297c5ba80936c4f6927633e5, server=f9fc96254ef8,44299,1731397650797 in 235 msec 2024-11-12T07:49:02,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-12T07:49:02,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a03352d0297c5ba80936c4f6927633e5, UNASSIGN in 240 msec 2024-11-12T07:49:02,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-12T07:49:02,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 244 msec 2024-11-12T07:49:02,512 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397742512"}]},"ts":"1731397742512"} 2024-11-12T07:49:02,513 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T07:49:02,513 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T07:49:02,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 254 msec 2024-11-12T07:49:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-12T07:49:02,578 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:02,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T07:49:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:02,581 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T07:49:02,582 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:02,584 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:49:02,586 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/recovered.edits] 2024-11-12T07:49:02,589 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/01e867d1de7845cd9086d969ad822042 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/01e867d1de7845cd9086d969ad822042 2024-11-12T07:49:02,590 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7cdde9ccda6d4f70b305cf67c20313b8 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/7cdde9ccda6d4f70b305cf67c20313b8 2024-11-12T07:49:02,591 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e101dea3aa934cc2b53045c0d937f3c3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/A/e101dea3aa934cc2b53045c0d937f3c3 2024-11-12T07:49:02,594 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/18136738f2a342c386843aed55e884a0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/18136738f2a342c386843aed55e884a0 2024-11-12T07:49:02,595 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/420db2ab17794e299d01d1a7d4a7e916 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/420db2ab17794e299d01d1a7d4a7e916 2024-11-12T07:49:02,596 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/78d1f920fb1e4da9bdc57830a07bc9d2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/B/78d1f920fb1e4da9bdc57830a07bc9d2 2024-11-12T07:49:02,598 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/13ed0d87a7ea4ebca5a52d87e4023997 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/13ed0d87a7ea4ebca5a52d87e4023997 2024-11-12T07:49:02,599 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/5127c39cd7564987ae3679ba0e98c8f3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/5127c39cd7564987ae3679ba0e98c8f3 2024-11-12T07:49:02,601 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/92086a82ee7c42d7a4eb8f7e63afa06a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/C/92086a82ee7c42d7a4eb8f7e63afa06a 2024-11-12T07:49:02,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/recovered.edits/353.seqid to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5/recovered.edits/353.seqid 2024-11-12T07:49:02,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/a03352d0297c5ba80936c4f6927633e5 2024-11-12T07:49:02,604 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T07:49:02,606 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:02,609 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T07:49:02,611 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T07:49:02,612 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:02,612 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T07:49:02,612 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731397742612"}]},"ts":"9223372036854775807"} 2024-11-12T07:49:02,614 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T07:49:02,614 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a03352d0297c5ba80936c4f6927633e5, NAME => 'TestAcidGuarantees,,1731397716601.a03352d0297c5ba80936c4f6927633e5.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T07:49:02,614 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T07:49:02,614 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731397742614"}]},"ts":"9223372036854775807"} 2024-11-12T07:49:02,616 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T07:49:02,617 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:02,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-12T07:49:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-12T07:49:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T07:49:02,689 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:02,703 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1287358721_22 at /127.0.0.1:57732 [Waiting for operation #291] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_462226526_22 at /127.0.0.1:49314 [Waiting for operation #211] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=396 (was 432), ProcessCount=11 (was 11), AvailableMemoryMB=5925 (was 5990) 2024-11-12T07:49:02,710 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=396, ProcessCount=11, AvailableMemoryMB=5924 2024-11-12T07:49:02,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:49:02,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:49:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:02,714 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:49:02,714 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:02,714 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-12T07:49:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T07:49:02,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:49:02,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742143_1319 (size=1061) 2024-11-12T07:49:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T07:49:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T07:49:03,128 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f 2024-11-12T07:49:03,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742144_1320 (size=53) 2024-11-12T07:49:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T07:49:03,574 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:49:03,574 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d56ec85ac1d426368f9bcec18d0d9e10, disabling compactions & flushes 2024-11-12T07:49:03,574 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:03,574 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:03,574 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. after waiting 0 ms 2024-11-12T07:49:03,574 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:03,574 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:03,574 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d56ec85ac1d426368f9bcec18d0d9e10: Waiting for close lock at 1731397743574Disabling compacts and flushes for region at 1731397743574Disabling writes for close at 1731397743574Writing region close event to WAL at 1731397743574Closed at 1731397743574 2024-11-12T07:49:03,577 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:49:03,577 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731397743577"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397743577"}]},"ts":"1731397743577"} 2024-11-12T07:49:03,580 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:49:03,582 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:49:03,583 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397743582"}]},"ts":"1731397743582"} 2024-11-12T07:49:03,584 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T07:49:03,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, ASSIGN}] 2024-11-12T07:49:03,585 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, ASSIGN 2024-11-12T07:49:03,586 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, ASSIGN; state=OFFLINE, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=false 2024-11-12T07:49:03,737 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d56ec85ac1d426368f9bcec18d0d9e10, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:03,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, ASSIGN because future has completed 2024-11-12T07:49:03,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:49:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T07:49:03,900 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:03,901 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:49:03,902 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,902 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:49:03,903 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,903 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,906 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,907 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:03,907 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56ec85ac1d426368f9bcec18d0d9e10 columnFamilyName A 2024-11-12T07:49:03,908 DEBUG [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:03,908 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(327): Store=d56ec85ac1d426368f9bcec18d0d9e10/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:03,908 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,909 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:03,909 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56ec85ac1d426368f9bcec18d0d9e10 columnFamilyName B 2024-11-12T07:49:03,909 DEBUG [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:03,909 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(327): Store=d56ec85ac1d426368f9bcec18d0d9e10/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:03,910 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,910 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:03,910 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56ec85ac1d426368f9bcec18d0d9e10 columnFamilyName C 2024-11-12T07:49:03,910 DEBUG [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:03,911 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(327): Store=d56ec85ac1d426368f9bcec18d0d9e10/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:03,911 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:03,911 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,912 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,912 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,913 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,913 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,913 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:49:03,914 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,915 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:49:03,915 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened d56ec85ac1d426368f9bcec18d0d9e10; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66175492, jitterRate=-0.013908326625823975}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:49:03,915 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:03,916 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for d56ec85ac1d426368f9bcec18d0d9e10: Running coprocessor pre-open hook at 1731397743903Writing region info on filesystem at 1731397743903Initializing all the Stores at 1731397743905 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397743905Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397743906 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397743906Cleaning up temporary data from old regions at 1731397743913 (+7 ms)Running coprocessor post-open hooks at 1731397743915 (+2 ms)Region opened successfully at 1731397743916 (+1 ms) 2024-11-12T07:49:03,917 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., pid=87, masterSystemTime=1731397743894 2024-11-12T07:49:03,918 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:03,919 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:03,919 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d56ec85ac1d426368f9bcec18d0d9e10, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:03,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:49:03,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-12T07:49:03,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 in 179 msec 2024-11-12T07:49:03,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-12T07:49:03,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, ASSIGN in 338 msec 2024-11-12T07:49:03,924 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:49:03,925 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397743924"}]},"ts":"1731397743924"} 2024-11-12T07:49:03,926 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T07:49:03,926 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:49:03,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2140 sec 2024-11-12T07:49:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T07:49:04,859 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:04,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:04,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:04,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:04,865 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2a80c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:04,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:04,869 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:04,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae48c4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:04,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:04,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:04,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:04,873 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:49:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,43319,1731397650023 2024-11-12T07:49:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7ab7af87 2024-11-12T07:49:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:49:04,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:49:04,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:49:04,876 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:49:04,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:04,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742145_1321 (size=1097) 2024-11-12T07:49:05,290 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T07:49:05,291 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T07:49:05,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:49:05,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, REOPEN/MOVE}] 2024-11-12T07:49:05,301 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, REOPEN/MOVE 2024-11-12T07:49:05,302 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d56ec85ac1d426368f9bcec18d0d9e10, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:05,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, REOPEN/MOVE because future has completed 2024-11-12T07:49:05,304 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:49:05,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:49:05,457 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,457 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:49:05,457 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing d56ec85ac1d426368f9bcec18d0d9e10, disabling compactions & flushes 2024-11-12T07:49:05,457 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:05,458 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:05,458 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. after waiting 0 ms 2024-11-12T07:49:05,458 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:05,461 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T07:49:05,462 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:05,462 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for d56ec85ac1d426368f9bcec18d0d9e10: Waiting for close lock at 1731397745457Running coprocessor pre-close hooks at 1731397745457Disabling compacts and flushes for region at 1731397745457Disabling writes for close at 1731397745458 (+1 ms)Writing region close event to WAL at 1731397745458Running coprocessor post-close hooks at 1731397745462 (+4 ms)Closed at 1731397745462 2024-11-12T07:49:05,462 WARN [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: d56ec85ac1d426368f9bcec18d0d9e10 to self. 2024-11-12T07:49:05,464 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d56ec85ac1d426368f9bcec18d0d9e10, regionState=CLOSED 2024-11-12T07:49:05,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:49:05,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-12T07:49:05,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 in 163 msec 2024-11-12T07:49:05,470 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, REOPEN/MOVE; state=CLOSED, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=true 2024-11-12T07:49:05,621 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d56ec85ac1d426368f9bcec18d0d9e10, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:05,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, REOPEN/MOVE because future has completed 2024-11-12T07:49:05,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:49:05,786 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:05,786 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:49:05,787 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,787 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:49:05,788 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,788 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,791 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,794 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:05,794 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56ec85ac1d426368f9bcec18d0d9e10 columnFamilyName A 2024-11-12T07:49:05,796 DEBUG [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:05,797 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(327): Store=d56ec85ac1d426368f9bcec18d0d9e10/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:05,797 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,798 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:05,798 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56ec85ac1d426368f9bcec18d0d9e10 columnFamilyName B 2024-11-12T07:49:05,798 DEBUG [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:05,799 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(327): Store=d56ec85ac1d426368f9bcec18d0d9e10/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:05,799 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,800 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:05,800 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d56ec85ac1d426368f9bcec18d0d9e10 columnFamilyName C 2024-11-12T07:49:05,800 DEBUG [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:05,801 INFO [StoreOpener-d56ec85ac1d426368f9bcec18d0d9e10-1 {}] regionserver.HStore(327): Store=d56ec85ac1d426368f9bcec18d0d9e10/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:05,801 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:05,801 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,802 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,803 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,804 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,804 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,804 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:49:05,805 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,806 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened d56ec85ac1d426368f9bcec18d0d9e10; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72296212, jitterRate=0.07729750871658325}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:49:05,806 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,806 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for d56ec85ac1d426368f9bcec18d0d9e10: Running coprocessor pre-open hook at 1731397745788Writing region info on filesystem at 1731397745788Initializing all the Stores at 1731397745790 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397745790Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397745791 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397745791Cleaning up temporary data from old regions at 1731397745804 (+13 ms)Running coprocessor post-open hooks at 1731397745806 (+2 ms)Region opened successfully at 1731397745806 2024-11-12T07:49:05,807 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., pid=92, masterSystemTime=1731397745778 2024-11-12T07:49:05,809 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:05,809 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:05,809 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d56ec85ac1d426368f9bcec18d0d9e10, regionState=OPEN, openSeqNum=5, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:05,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:49:05,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-12T07:49:05,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 in 187 msec 2024-11-12T07:49:05,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-12T07:49:05,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, REOPEN/MOVE in 512 msec 2024-11-12T07:49:05,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-12T07:49:05,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-11-12T07:49:05,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-12T07:49:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-12T07:49:05,819 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:05,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,820 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71629ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,821 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e05590e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,823 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a7b7897, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,824 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c144533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,826 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6048111f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1613856f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,828 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,828 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c802d0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,829 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@312011b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,831 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5f1baa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb958e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,833 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ccbd01f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb986bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,836 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e06f847, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,837 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355d585a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,839 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14208ab5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,840 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58afe417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,842 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b332893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,843 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5c4162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:05,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:05,845 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be06714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:05,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:05,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:05,846 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@235eba97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:05,847 DEBUG [Thread-1478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,847 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,847 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,847 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,848 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,848 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,849 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,849 DEBUG [Thread-1472 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,850 DEBUG [Thread-1476 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-12T07:49:05,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T07:49:05,852 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:05,853 DEBUG [Thread-1474 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:05,853 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:05,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:05,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:05,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:05,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,858 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:05,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:49:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:05,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:49:05,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:05,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:05,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:05,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:05,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:05,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:05,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112593e02dcc1f2448e80a4e973163b06b0_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397745862/Put/seqid=0 2024-11-12T07:49:05,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742146_1322 (size=17034) 2024-11-12T07:49:05,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397805891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:05,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397805892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:05,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:05,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397805898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:05,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397805899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:05,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397805902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T07:49:06,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T07:49:06,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:06,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397806008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397806008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397806008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397806018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397806018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T07:49:06,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:06,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T07:49:06,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397806218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397806218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397806218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397806238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397806238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:06,303 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112593e02dcc1f2448e80a4e973163b06b0_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112593e02dcc1f2448e80a4e973163b06b0_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:06,303 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/e8089dcb0d654356ad69728776022c75, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:06,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/e8089dcb0d654356ad69728776022c75 is 175, key is test_row_0/A:col10/1731397745862/Put/seqid=0 2024-11-12T07:49:06,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742147_1323 (size=48139) 2024-11-12T07:49:06,310 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/e8089dcb0d654356ad69728776022c75 2024-11-12T07:49:06,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T07:49:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/86a4386297634e89a873e34393131c3e is 50, key is test_row_0/B:col10/1731397745862/Put/seqid=0 2024-11-12T07:49:06,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742148_1324 (size=12001) 2024-11-12T07:49:06,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/86a4386297634e89a873e34393131c3e 2024-11-12T07:49:06,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c5ada7618a904cffa02d1b83814ce920 is 50, key is test_row_0/C:col10/1731397745862/Put/seqid=0 2024-11-12T07:49:06,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742149_1325 (size=12001) 2024-11-12T07:49:06,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T07:49:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T07:49:06,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397806528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397806538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397806538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397806547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:06,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397806547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:06,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:06,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:06,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T07:49:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c5ada7618a904cffa02d1b83814ce920 2024-11-12T07:49:06,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/e8089dcb0d654356ad69728776022c75 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e8089dcb0d654356ad69728776022c75 2024-11-12T07:49:06,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T07:49:06,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:06,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:06,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e8089dcb0d654356ad69728776022c75, entries=250, sequenceid=16, filesize=47.0 K 2024-11-12T07:49:06,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/86a4386297634e89a873e34393131c3e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/86a4386297634e89a873e34393131c3e 2024-11-12T07:49:06,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/86a4386297634e89a873e34393131c3e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T07:49:06,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c5ada7618a904cffa02d1b83814ce920 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5ada7618a904cffa02d1b83814ce920 2024-11-12T07:49:06,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5ada7618a904cffa02d1b83814ce920, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T07:49:06,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d56ec85ac1d426368f9bcec18d0d9e10 in 918ms, sequenceid=16, compaction requested=false 2024-11-12T07:49:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T07:49:06,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:06,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T07:49:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:06,924 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T07:49:06,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:06,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:06,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:06,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:06,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:06,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:06,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111210292a10ad854558b739a890b4f5e3b5_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397745896/Put/seqid=0 2024-11-12T07:49:06,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742150_1326 (size=12154) 2024-11-12T07:49:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T07:49:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:07,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:07,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397807058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397807058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397807059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397807060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397807063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397807168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397807168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397807178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:07,340 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111210292a10ad854558b739a890b4f5e3b5_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111210292a10ad854558b739a890b4f5e3b5_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/0671c41696b944b59abf78f902f17364, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:07,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/0671c41696b944b59abf78f902f17364 is 175, key is test_row_0/A:col10/1731397745896/Put/seqid=0 2024-11-12T07:49:07,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742151_1327 (size=30955) 2024-11-12T07:49:07,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397807378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397807378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397807388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397807687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397807687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:07,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397807698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:07,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/0671c41696b944b59abf78f902f17364 2024-11-12T07:49:07,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/8439dea76ee14c33844fa63a53f56916 is 50, key is test_row_0/B:col10/1731397745896/Put/seqid=0 2024-11-12T07:49:07,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742152_1328 (size=12001) 2024-11-12T07:49:07,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T07:49:08,026 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:49:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397808068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:08,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397808068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:08,077 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in d56ec85ac1d426368f9bcec18d0d9e10 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:08,077 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in d56ec85ac1d426368f9bcec18d0d9e10 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:08,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:08,154 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/8439dea76ee14c33844fa63a53f56916 2024-11-12T07:49:08,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397808197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:08,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/6dc149a1509648a0a0dd53fdb5c309e9 is 50, key is test_row_0/C:col10/1731397745896/Put/seqid=0 2024-11-12T07:49:08,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742153_1329 (size=12001) 2024-11-12T07:49:08,211 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/6dc149a1509648a0a0dd53fdb5c309e9 2024-11-12T07:49:08,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397808208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/0671c41696b944b59abf78f902f17364 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/0671c41696b944b59abf78f902f17364 2024-11-12T07:49:08,217 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/0671c41696b944b59abf78f902f17364, entries=150, sequenceid=36, filesize=30.2 K 2024-11-12T07:49:08,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/8439dea76ee14c33844fa63a53f56916 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/8439dea76ee14c33844fa63a53f56916 2024-11-12T07:49:08,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397808218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:08,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:08,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:08,225 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/8439dea76ee14c33844fa63a53f56916, entries=150, sequenceid=36, filesize=11.7 K 2024-11-12T07:49:08,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/6dc149a1509648a0a0dd53fdb5c309e9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6dc149a1509648a0a0dd53fdb5c309e9 2024-11-12T07:49:08,229 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6dc149a1509648a0a0dd53fdb5c309e9, entries=150, sequenceid=36, filesize=11.7 K 2024-11-12T07:49:08,230 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d56ec85ac1d426368f9bcec18d0d9e10 in 1306ms, sequenceid=36, compaction requested=false 2024-11-12T07:49:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:08,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-12T07:49:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-12T07:49:08,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-12T07:49:08,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3780 sec 2024-11-12T07:49:08,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.3840 sec 2024-11-12T07:49:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:09,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T07:49:09,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:09,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:09,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:09,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:09,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:09,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:09,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d9c6faac9cea4b4dab204d63c05f1947_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397747059/Put/seqid=0 2024-11-12T07:49:09,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742154_1330 (size=14594) 2024-11-12T07:49:09,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397809263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397809264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397809269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397809378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397809378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397809378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397809588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397809588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397809588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,620 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:09,624 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d9c6faac9cea4b4dab204d63c05f1947_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9c6faac9cea4b4dab204d63c05f1947_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:09,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/04ece208b2894f52b8c5de4a60a79833, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:09,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/04ece208b2894f52b8c5de4a60a79833 is 175, key is test_row_0/A:col10/1731397747059/Put/seqid=0 2024-11-12T07:49:09,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742155_1331 (size=39549) 2024-11-12T07:49:09,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397809897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397809898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397809898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:09,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:09,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:09,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T07:49:10,018 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:10,019 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-12T07:49:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T07:49:10,020 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:10,021 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:10,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:10,030 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/04ece208b2894f52b8c5de4a60a79833 2024-11-12T07:49:10,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/5edafaadef8d47bdb83f001d9d531f59 is 50, key is test_row_0/B:col10/1731397747059/Put/seqid=0 2024-11-12T07:49:10,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742156_1332 (size=12001) 2024-11-12T07:49:10,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397810098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:10,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397810098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:10,108 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in d56ec85ac1d426368f9bcec18d0d9e10 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:10,108 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in d56ec85ac1d426368f9bcec18d0d9e10 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T07:49:10,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T07:49:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:10,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T07:49:10,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T07:49:10,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397810408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:10,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397810408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:10,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397810417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:10,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/5edafaadef8d47bdb83f001d9d531f59 2024-11-12T07:49:10,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c5758c63b6fb4f048c297279b497d330 is 50, key is test_row_0/C:col10/1731397747059/Put/seqid=0 2024-11-12T07:49:10,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742157_1333 (size=12001) 2024-11-12T07:49:10,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T07:49:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:10,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T07:49:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T07:49:10,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T07:49:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:10,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c5758c63b6fb4f048c297279b497d330 2024-11-12T07:49:10,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/04ece208b2894f52b8c5de4a60a79833 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/04ece208b2894f52b8c5de4a60a79833 2024-11-12T07:49:10,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/04ece208b2894f52b8c5de4a60a79833, entries=200, sequenceid=49, filesize=38.6 K 2024-11-12T07:49:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/5edafaadef8d47bdb83f001d9d531f59 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5edafaadef8d47bdb83f001d9d531f59 2024-11-12T07:49:10,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5edafaadef8d47bdb83f001d9d531f59, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T07:49:10,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c5758c63b6fb4f048c297279b497d330 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5758c63b6fb4f048c297279b497d330 2024-11-12T07:49:10,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5758c63b6fb4f048c297279b497d330, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T07:49:10,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d56ec85ac1d426368f9bcec18d0d9e10 in 1662ms, sequenceid=49, compaction requested=true 2024-11-12T07:49:10,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:10,872 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:10,872 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:10,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:10,873 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:10,873 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/A is initiating minor compaction (all files) 2024-11-12T07:49:10,873 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/A in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,873 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e8089dcb0d654356ad69728776022c75, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/0671c41696b944b59abf78f902f17364, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/04ece208b2894f52b8c5de4a60a79833] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=115.9 K 2024-11-12T07:49:10,873 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,873 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e8089dcb0d654356ad69728776022c75, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/0671c41696b944b59abf78f902f17364, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/04ece208b2894f52b8c5de4a60a79833] 2024-11-12T07:49:10,873 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:10,873 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/B is initiating minor compaction (all files) 2024-11-12T07:49:10,873 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/B in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,873 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/86a4386297634e89a873e34393131c3e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/8439dea76ee14c33844fa63a53f56916, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5edafaadef8d47bdb83f001d9d531f59] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=35.2 K 2024-11-12T07:49:10,873 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8089dcb0d654356ad69728776022c75, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731397745858 2024-11-12T07:49:10,873 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 86a4386297634e89a873e34393131c3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731397745862 2024-11-12T07:49:10,873 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0671c41696b944b59abf78f902f17364, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731397745896 2024-11-12T07:49:10,874 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8439dea76ee14c33844fa63a53f56916, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731397745896 2024-11-12T07:49:10,874 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ece208b2894f52b8c5de4a60a79833, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731397747058 2024-11-12T07:49:10,874 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5edafaadef8d47bdb83f001d9d531f59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731397747058 2024-11-12T07:49:10,878 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:10,880 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111276b4d6230fcf4ae0b9983ebef01d6887_d56ec85ac1d426368f9bcec18d0d9e10 store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:10,880 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:10,881 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/60e02c8e6b1f474c8540c781de8e88a3 is 50, key is test_row_0/B:col10/1731397747059/Put/seqid=0 2024-11-12T07:49:10,882 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111276b4d6230fcf4ae0b9983ebef01d6887_d56ec85ac1d426368f9bcec18d0d9e10, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:10,882 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111276b4d6230fcf4ae0b9983ebef01d6887_d56ec85ac1d426368f9bcec18d0d9e10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:10,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742158_1334 (size=12104) 2024-11-12T07:49:10,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742159_1335 (size=4469) 2024-11-12T07:49:10,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T07:49:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:10,938 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T07:49:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:10,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:10,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112422150d104eb4930b2272a2aad0e66b8_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397749261/Put/seqid=0 2024-11-12T07:49:10,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742160_1336 (size=12154) 2024-11-12T07:49:10,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:10,954 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112422150d104eb4930b2272a2aad0e66b8_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112422150d104eb4930b2272a2aad0e66b8_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:10,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ecb36764c93d4e4c838af82417fc3195, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:10,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ecb36764c93d4e4c838af82417fc3195 is 175, key is test_row_0/A:col10/1731397749261/Put/seqid=0 2024-11-12T07:49:10,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742161_1337 (size=30955) 2024-11-12T07:49:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T07:49:11,289 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/60e02c8e6b1f474c8540c781de8e88a3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/60e02c8e6b1f474c8540c781de8e88a3 2024-11-12T07:49:11,290 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#A#compaction#285 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:11,291 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/3695f4bdb29d4800afc702fa2ed1eac9 is 175, key is test_row_0/A:col10/1731397747059/Put/seqid=0 2024-11-12T07:49:11,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742162_1338 (size=31058) 2024-11-12T07:49:11,295 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/B of d56ec85ac1d426368f9bcec18d0d9e10 into 60e02c8e6b1f474c8540c781de8e88a3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:11,295 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:11,295 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/B, priority=13, startTime=1731397750872; duration=0sec 2024-11-12T07:49:11,295 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:11,295 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:B 2024-11-12T07:49:11,295 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:11,296 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:11,297 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/C is initiating minor compaction (all files) 2024-11-12T07:49:11,297 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/C in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:11,297 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5ada7618a904cffa02d1b83814ce920, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6dc149a1509648a0a0dd53fdb5c309e9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5758c63b6fb4f048c297279b497d330] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=35.2 K 2024-11-12T07:49:11,297 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ada7618a904cffa02d1b83814ce920, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731397745862 2024-11-12T07:49:11,297 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc149a1509648a0a0dd53fdb5c309e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731397745896 2024-11-12T07:49:11,297 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c5758c63b6fb4f048c297279b497d330, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731397747058 2024-11-12T07:49:11,304 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:11,304 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/38f63e0166714daea3cb1965464575b9 is 50, key is test_row_0/C:col10/1731397747059/Put/seqid=0 2024-11-12T07:49:11,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742163_1339 (size=12104) 2024-11-12T07:49:11,359 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ecb36764c93d4e4c838af82417fc3195 2024-11-12T07:49:11,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/bab091cb300241b4ac8814846a7c7aee is 50, key is test_row_0/B:col10/1731397749261/Put/seqid=0 2024-11-12T07:49:11,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742164_1340 (size=12001) 2024-11-12T07:49:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:11,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:11,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397811458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:11,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397811459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:11,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397811461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:11,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397811567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397811578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397811578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:11,699 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/3695f4bdb29d4800afc702fa2ed1eac9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3695f4bdb29d4800afc702fa2ed1eac9 2024-11-12T07:49:11,704 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/A of d56ec85ac1d426368f9bcec18d0d9e10 into 3695f4bdb29d4800afc702fa2ed1eac9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:11,704 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:11,704 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/A, priority=13, startTime=1731397750872; duration=0sec 2024-11-12T07:49:11,704 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:11,704 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:A 2024-11-12T07:49:11,713 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/38f63e0166714daea3cb1965464575b9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/38f63e0166714daea3cb1965464575b9 2024-11-12T07:49:11,717 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/C of d56ec85ac1d426368f9bcec18d0d9e10 into 38f63e0166714daea3cb1965464575b9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:11,717 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:11,717 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/C, priority=13, startTime=1731397750872; duration=0sec 2024-11-12T07:49:11,717 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:11,717 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:C 2024-11-12T07:49:11,768 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/bab091cb300241b4ac8814846a7c7aee 2024-11-12T07:49:11,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/64c433ee8076403a94f93202eb30f95e is 50, key is test_row_0/C:col10/1731397749261/Put/seqid=0 2024-11-12T07:49:11,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742165_1341 (size=12001) 2024-11-12T07:49:11,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397811777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:11,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397811788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397811788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397812088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397812098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397812098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T07:49:12,178 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/64c433ee8076403a94f93202eb30f95e 2024-11-12T07:49:12,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ecb36764c93d4e4c838af82417fc3195 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ecb36764c93d4e4c838af82417fc3195 2024-11-12T07:49:12,185 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ecb36764c93d4e4c838af82417fc3195, entries=150, sequenceid=68, filesize=30.2 K 2024-11-12T07:49:12,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/bab091cb300241b4ac8814846a7c7aee as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bab091cb300241b4ac8814846a7c7aee 2024-11-12T07:49:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,190 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bab091cb300241b4ac8814846a7c7aee, entries=150, sequenceid=68, filesize=11.7 K 2024-11-12T07:49:12,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/64c433ee8076403a94f93202eb30f95e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/64c433ee8076403a94f93202eb30f95e 2024-11-12T07:49:12,194 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/64c433ee8076403a94f93202eb30f95e, entries=150, sequenceid=68, filesize=11.7 K 2024-11-12T07:49:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,195 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d56ec85ac1d426368f9bcec18d0d9e10 in 1257ms, sequenceid=68, compaction requested=false 2024-11-12T07:49:12,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:12,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:12,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-12T07:49:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-12T07:49:12,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-12T07:49:12,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-12T07:49:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.1800 sec 2024-11-12T07:49:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:49:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111216665244a1c647e39e56400ff24dfc60_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_1/A:col10/1731397752621/Put/seqid=0 2024-11-12T07:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742166_1342 (size=12154) 2024-11-12T07:49:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,652 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111216665244a1c647e39e56400ff24dfc60_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111216665244a1c647e39e56400ff24dfc60_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:12,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/c8a6a6c93556494e9cbc80555079ea8a, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:12,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/c8a6a6c93556494e9cbc80555079ea8a is 175, key is test_row_1/A:col10/1731397752621/Put/seqid=0 2024-11-12T07:49:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:12,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742167_1343 (size=30951) 2024-11-12T07:49:12,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/c8a6a6c93556494e9cbc80555079ea8a 2024-11-12T07:49:12,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/543b5d65f570431c81051a5caa8aab77 is 50, key is test_row_1/B:col10/1731397752621/Put/seqid=0 2024-11-12T07:49:12,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742168_1344 (size=9657) 2024-11-12T07:49:12,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397812705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397812711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397812712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397812827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397812828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397812828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397813047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397813048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397813048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/543b5d65f570431c81051a5caa8aab77 2024-11-12T07:49:13,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/e87d47da7e1f431db9e663fad37fa8da is 50, key is test_row_1/C:col10/1731397752621/Put/seqid=0 2024-11-12T07:49:13,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742169_1345 (size=9657) 2024-11-12T07:49:13,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/e87d47da7e1f431db9e663fad37fa8da 2024-11-12T07:49:13,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/c8a6a6c93556494e9cbc80555079ea8a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c8a6a6c93556494e9cbc80555079ea8a 2024-11-12T07:49:13,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c8a6a6c93556494e9cbc80555079ea8a, entries=150, sequenceid=85, filesize=30.2 K 2024-11-12T07:49:13,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/543b5d65f570431c81051a5caa8aab77 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/543b5d65f570431c81051a5caa8aab77 2024-11-12T07:49:13,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/543b5d65f570431c81051a5caa8aab77, entries=100, sequenceid=85, filesize=9.4 K 2024-11-12T07:49:13,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/e87d47da7e1f431db9e663fad37fa8da as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e87d47da7e1f431db9e663fad37fa8da 2024-11-12T07:49:13,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e87d47da7e1f431db9e663fad37fa8da, entries=100, sequenceid=85, filesize=9.4 K 2024-11-12T07:49:13,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d56ec85ac1d426368f9bcec18d0d9e10 in 498ms, sequenceid=85, compaction requested=true 2024-11-12T07:49:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:13,120 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:13,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:13,121 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:13,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:13,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:13,123 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:13,123 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/B is initiating minor compaction (all files) 2024-11-12T07:49:13,123 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/B in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:13,123 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/60e02c8e6b1f474c8540c781de8e88a3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bab091cb300241b4ac8814846a7c7aee, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/543b5d65f570431c81051a5caa8aab77] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=33.0 K 2024-11-12T07:49:13,123 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:13,124 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/A is initiating minor compaction (all files) 2024-11-12T07:49:13,124 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/A in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:13,124 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3695f4bdb29d4800afc702fa2ed1eac9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ecb36764c93d4e4c838af82417fc3195, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c8a6a6c93556494e9cbc80555079ea8a] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=90.8 K 2024-11-12T07:49:13,124 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:13,124 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3695f4bdb29d4800afc702fa2ed1eac9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ecb36764c93d4e4c838af82417fc3195, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c8a6a6c93556494e9cbc80555079ea8a] 2024-11-12T07:49:13,124 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e02c8e6b1f474c8540c781de8e88a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731397747058 2024-11-12T07:49:13,124 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3695f4bdb29d4800afc702fa2ed1eac9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731397747058 2024-11-12T07:49:13,124 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting bab091cb300241b4ac8814846a7c7aee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731397749243 2024-11-12T07:49:13,124 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb36764c93d4e4c838af82417fc3195, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731397749243 2024-11-12T07:49:13,124 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 543b5d65f570431c81051a5caa8aab77, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731397751457 2024-11-12T07:49:13,125 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8a6a6c93556494e9cbc80555079ea8a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731397751457 2024-11-12T07:49:13,130 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:13,131 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/fe69f39511cf4c00887d4d0e68a2f3dc is 50, key is test_row_0/B:col10/1731397749261/Put/seqid=0 2024-11-12T07:49:13,135 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:13,139 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111291743cbf3a3e4d2385e12bcb5f723e00_d56ec85ac1d426368f9bcec18d0d9e10 store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:13,141 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111291743cbf3a3e4d2385e12bcb5f723e00_d56ec85ac1d426368f9bcec18d0d9e10, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:13,141 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111291743cbf3a3e4d2385e12bcb5f723e00_d56ec85ac1d426368f9bcec18d0d9e10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:13,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742170_1346 (size=12207) 2024-11-12T07:49:13,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742171_1347 (size=4469) 2024-11-12T07:49:13,152 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#A#compaction#295 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:13,152 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/6cf06cea2410475d964aa11f230f8cac is 175, key is test_row_0/A:col10/1731397749261/Put/seqid=0 2024-11-12T07:49:13,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742172_1348 (size=31268) 2024-11-12T07:49:13,202 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/6cf06cea2410475d964aa11f230f8cac as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/6cf06cea2410475d964aa11f230f8cac 2024-11-12T07:49:13,206 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/A of d56ec85ac1d426368f9bcec18d0d9e10 into 6cf06cea2410475d964aa11f230f8cac(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:13,206 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:13,206 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/A, priority=13, startTime=1731397753120; duration=0sec 2024-11-12T07:49:13,206 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:13,206 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:A 2024-11-12T07:49:13,206 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:13,207 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:13,207 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/C is initiating minor compaction (all files) 2024-11-12T07:49:13,207 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/C in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:13,207 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/38f63e0166714daea3cb1965464575b9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/64c433ee8076403a94f93202eb30f95e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e87d47da7e1f431db9e663fad37fa8da] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=33.0 K 2024-11-12T07:49:13,207 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f63e0166714daea3cb1965464575b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731397747058 2024-11-12T07:49:13,207 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c433ee8076403a94f93202eb30f95e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731397749243 2024-11-12T07:49:13,208 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e87d47da7e1f431db9e663fad37fa8da, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731397751457 2024-11-12T07:49:13,214 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:13,215 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/acc00eea611645c59a6e781bc2b1791d is 50, key is test_row_0/C:col10/1731397749261/Put/seqid=0 2024-11-12T07:49:13,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742173_1349 (size=12207) 2024-11-12T07:49:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:13,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T07:49:13,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:13,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:13,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:13,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:13,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:13,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:13,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dfe71421e4054b2b8dd6f6dc06e1b728_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397752703/Put/seqid=0 2024-11-12T07:49:13,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742174_1350 (size=14594) 2024-11-12T07:49:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397813386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397813386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397813387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397813497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397813497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397813498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,550 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/fe69f39511cf4c00887d4d0e68a2f3dc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/fe69f39511cf4c00887d4d0e68a2f3dc 2024-11-12T07:49:13,555 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/B of d56ec85ac1d426368f9bcec18d0d9e10 into fe69f39511cf4c00887d4d0e68a2f3dc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:13,555 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:13,556 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/B, priority=13, startTime=1731397753120; duration=0sec 2024-11-12T07:49:13,556 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:13,556 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:B 2024-11-12T07:49:13,630 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/acc00eea611645c59a6e781bc2b1791d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/acc00eea611645c59a6e781bc2b1791d 2024-11-12T07:49:13,635 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/C of d56ec85ac1d426368f9bcec18d0d9e10 into acc00eea611645c59a6e781bc2b1791d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:13,635 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:13,635 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/C, priority=13, startTime=1731397753121; duration=0sec 2024-11-12T07:49:13,635 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:13,635 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:C 2024-11-12T07:49:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397813707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397813707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397813708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:13,774 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:13,777 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dfe71421e4054b2b8dd6f6dc06e1b728_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dfe71421e4054b2b8dd6f6dc06e1b728_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:13,777 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ef9062e266d5429d81506c2ad29a9637, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:13,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ef9062e266d5429d81506c2ad29a9637 is 175, key is test_row_0/A:col10/1731397752703/Put/seqid=0 2024-11-12T07:49:13,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742175_1351 (size=39549) 2024-11-12T07:49:13,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ef9062e266d5429d81506c2ad29a9637 2024-11-12T07:49:13,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/5908ccbbe40b436a96842b7bd9e8e170 is 50, key is test_row_0/B:col10/1731397752703/Put/seqid=0 2024-11-12T07:49:13,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742176_1352 (size=12001) 2024-11-12T07:49:14,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397814018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:14,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397814028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:14,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397814028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:14,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397814118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:14,124 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in d56ec85ac1d426368f9bcec18d0d9e10 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:14,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397814137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:14,142 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in d56ec85ac1d426368f9bcec18d0d9e10 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T07:49:14,188 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:14,188 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-12T07:49:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T07:49:14,190 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:14,191 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:14,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:14,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/5908ccbbe40b436a96842b7bd9e8e170 2024-11-12T07:49:14,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/57f90bf579ce4265bdcfda2470537687 is 50, key is test_row_0/C:col10/1731397752703/Put/seqid=0 2024-11-12T07:49:14,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742177_1353 (size=12001) 2024-11-12T07:49:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T07:49:14,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T07:49:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:14,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:14,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T07:49:14,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:14,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:14,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:14,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:14,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T07:49:14,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397814527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397814538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397814538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:14,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/57f90bf579ce4265bdcfda2470537687 2024-11-12T07:49:14,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ef9062e266d5429d81506c2ad29a9637 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ef9062e266d5429d81506c2ad29a9637 2024-11-12T07:49:14,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ef9062e266d5429d81506c2ad29a9637, entries=200, sequenceid=105, filesize=38.6 K 2024-11-12T07:49:14,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/5908ccbbe40b436a96842b7bd9e8e170 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5908ccbbe40b436a96842b7bd9e8e170 2024-11-12T07:49:14,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5908ccbbe40b436a96842b7bd9e8e170, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T07:49:14,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/57f90bf579ce4265bdcfda2470537687 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/57f90bf579ce4265bdcfda2470537687 2024-11-12T07:49:14,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/57f90bf579ce4265bdcfda2470537687, entries=150, sequenceid=105, filesize=11.7 K 2024-11-12T07:49:14,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d56ec85ac1d426368f9bcec18d0d9e10 in 1259ms, sequenceid=105, compaction requested=false 2024-11-12T07:49:14,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:14,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T07:49:14,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:14,649 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T07:49:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:14,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:14,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111221a7b9410311461a83225603a0664b9b_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397753385/Put/seqid=0 2024-11-12T07:49:14,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742178_1354 (size=12154) 2024-11-12T07:49:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T07:49:15,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:15,061 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111221a7b9410311461a83225603a0664b9b_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111221a7b9410311461a83225603a0664b9b_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:15,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/39ce90d289e246e4bbf98e8083f3c211, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:15,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/39ce90d289e246e4bbf98e8083f3c211 is 175, key is test_row_0/A:col10/1731397753385/Put/seqid=0 2024-11-12T07:49:15,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742179_1355 (size=30955) 2024-11-12T07:49:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T07:49:15,467 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/39ce90d289e246e4bbf98e8083f3c211 2024-11-12T07:49:15,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/4bd781bddca041228a9379c006cc58fa is 50, key is test_row_0/B:col10/1731397753385/Put/seqid=0 2024-11-12T07:49:15,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742180_1356 (size=12001) 2024-11-12T07:49:15,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:15,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397815577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397815586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:15,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397815587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397815688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:15,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397815698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:15,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397815708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:15,877 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/4bd781bddca041228a9379c006cc58fa 2024-11-12T07:49:15,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/e58189e7dcd448909fe7b9ab3a63d049 is 50, key is test_row_0/C:col10/1731397753385/Put/seqid=0 2024-11-12T07:49:15,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742181_1357 (size=12001) 2024-11-12T07:49:15,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397815897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:15,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397815918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:15,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397815928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397816218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397816237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397816248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,287 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/e58189e7dcd448909fe7b9ab3a63d049 2024-11-12T07:49:16,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/39ce90d289e246e4bbf98e8083f3c211 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/39ce90d289e246e4bbf98e8083f3c211 2024-11-12T07:49:16,294 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/39ce90d289e246e4bbf98e8083f3c211, entries=150, sequenceid=120, filesize=30.2 K 2024-11-12T07:49:16,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/4bd781bddca041228a9379c006cc58fa as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4bd781bddca041228a9379c006cc58fa 2024-11-12T07:49:16,297 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4bd781bddca041228a9379c006cc58fa, entries=150, sequenceid=120, filesize=11.7 K 2024-11-12T07:49:16,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/e58189e7dcd448909fe7b9ab3a63d049 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e58189e7dcd448909fe7b9ab3a63d049 2024-11-12T07:49:16,301 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e58189e7dcd448909fe7b9ab3a63d049, entries=150, sequenceid=120, filesize=11.7 K 2024-11-12T07:49:16,302 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for d56ec85ac1d426368f9bcec18d0d9e10 in 1653ms, sequenceid=120, compaction requested=true 2024-11-12T07:49:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:16,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-12T07:49:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-12T07:49:16,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-12T07:49:16,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1120 sec 2024-11-12T07:49:16,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.1170 sec 2024-11-12T07:49:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T07:49:16,338 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:16,338 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-12T07:49:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T07:49:16,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:16,340 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:16,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T07:49:16,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T07:49:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:16,492 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T07:49:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:16,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:16,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:16,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:16,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111273d0e31f793d48b7a4be58064232e573_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397755585/Put/seqid=0 2024-11-12T07:49:16,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742182_1358 (size=12304) 2024-11-12T07:49:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:16,540 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111273d0e31f793d48b7a4be58064232e573_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111273d0e31f793d48b7a4be58064232e573_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ee1541bed3074a7f8a3086175b7e541e, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:16,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ee1541bed3074a7f8a3086175b7e541e is 175, key is test_row_0/A:col10/1731397755585/Put/seqid=0 2024-11-12T07:49:16,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742183_1359 (size=31105) 2024-11-12T07:49:16,561 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ee1541bed3074a7f8a3086175b7e541e 2024-11-12T07:49:16,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/e1a6e46d8bd34db0ad72678a56ff45dc is 50, key is test_row_0/B:col10/1731397755585/Put/seqid=0 2024-11-12T07:49:16,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742184_1360 (size=12151) 2024-11-12T07:49:16,607 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/e1a6e46d8bd34db0ad72678a56ff45dc 2024-11-12T07:49:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c6367defe8284936b6004912c21d63a9 is 50, key is test_row_0/C:col10/1731397755585/Put/seqid=0 2024-11-12T07:49:16,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742185_1361 (size=12151) 2024-11-12T07:49:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T07:49:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:16,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:16,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397816765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397816767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397816769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397816877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397816888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:16,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397816888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:16,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T07:49:17,029 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c6367defe8284936b6004912c21d63a9 2024-11-12T07:49:17,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/ee1541bed3074a7f8a3086175b7e541e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ee1541bed3074a7f8a3086175b7e541e 2024-11-12T07:49:17,035 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ee1541bed3074a7f8a3086175b7e541e, entries=150, sequenceid=140, filesize=30.4 K 2024-11-12T07:49:17,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/e1a6e46d8bd34db0ad72678a56ff45dc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/e1a6e46d8bd34db0ad72678a56ff45dc 2024-11-12T07:49:17,039 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/e1a6e46d8bd34db0ad72678a56ff45dc, entries=150, sequenceid=140, filesize=11.9 K 2024-11-12T07:49:17,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/c6367defe8284936b6004912c21d63a9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c6367defe8284936b6004912c21d63a9 2024-11-12T07:49:17,044 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c6367defe8284936b6004912c21d63a9, entries=150, sequenceid=140, filesize=11.9 K 2024-11-12T07:49:17,045 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d56ec85ac1d426368f9bcec18d0d9e10 in 553ms, sequenceid=140, compaction requested=true 2024-11-12T07:49:17,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:17,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:17,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-12T07:49:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-12T07:49:17,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-12T07:49:17,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 706 msec 2024-11-12T07:49:17,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 710 msec 2024-11-12T07:49:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:17,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T07:49:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:17,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:17,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111285389de7e0254790a314c436017b5d9c_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397757092/Put/seqid=0 2024-11-12T07:49:17,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742186_1362 (size=14794) 2024-11-12T07:49:17,104 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:17,107 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111285389de7e0254790a314c436017b5d9c_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285389de7e0254790a314c436017b5d9c_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:17,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/544a12c101f946d9a5d1310bfac0fadc, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:17,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/544a12c101f946d9a5d1310bfac0fadc is 175, key is test_row_0/A:col10/1731397757092/Put/seqid=0 2024-11-12T07:49:17,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742187_1363 (size=39749) 2024-11-12T07:49:17,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/544a12c101f946d9a5d1310bfac0fadc 2024-11-12T07:49:17,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/3558e7e9f78e41f19320e09013afa4f4 is 50, key is test_row_0/B:col10/1731397757092/Put/seqid=0 2024-11-12T07:49:17,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742188_1364 (size=12151) 2024-11-12T07:49:17,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/3558e7e9f78e41f19320e09013afa4f4 2024-11-12T07:49:17,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/816a51ed90d0436988b9a46178324522 is 50, key is test_row_0/C:col10/1731397757092/Put/seqid=0 2024-11-12T07:49:17,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742189_1365 (size=12151) 2024-11-12T07:49:17,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397817161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397817165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397817166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397817278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397817287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397817287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T07:49:17,478 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:17,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-12T07:49:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T07:49:17,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:17,481 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:17,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:17,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397817488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397817507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397817507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/816a51ed90d0436988b9a46178324522 2024-11-12T07:49:17,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/544a12c101f946d9a5d1310bfac0fadc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/544a12c101f946d9a5d1310bfac0fadc 2024-11-12T07:49:17,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/544a12c101f946d9a5d1310bfac0fadc, entries=200, sequenceid=153, filesize=38.8 K 2024-11-12T07:49:17,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/3558e7e9f78e41f19320e09013afa4f4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/3558e7e9f78e41f19320e09013afa4f4 2024-11-12T07:49:17,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/3558e7e9f78e41f19320e09013afa4f4, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T07:49:17,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/816a51ed90d0436988b9a46178324522 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/816a51ed90d0436988b9a46178324522 2024-11-12T07:49:17,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/816a51ed90d0436988b9a46178324522, entries=150, sequenceid=153, filesize=11.9 K 2024-11-12T07:49:17,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d56ec85ac1d426368f9bcec18d0d9e10 in 466ms, sequenceid=153, compaction requested=true 2024-11-12T07:49:17,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:17,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:17,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:17,559 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T07:49:17,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:17,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:17,559 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T07:49:17,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:17,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:17,560 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T07:49:17,560 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172626 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T07:49:17,560 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/A is initiating minor compaction (all files) 2024-11-12T07:49:17,560 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/B is initiating minor compaction (all files) 2024-11-12T07:49:17,560 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/A in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:17,560 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/B in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:17,561 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/fe69f39511cf4c00887d4d0e68a2f3dc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5908ccbbe40b436a96842b7bd9e8e170, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4bd781bddca041228a9379c006cc58fa, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/e1a6e46d8bd34db0ad72678a56ff45dc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/3558e7e9f78e41f19320e09013afa4f4] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=59.1 K 2024-11-12T07:49:17,561 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/6cf06cea2410475d964aa11f230f8cac, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ef9062e266d5429d81506c2ad29a9637, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/39ce90d289e246e4bbf98e8083f3c211, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ee1541bed3074a7f8a3086175b7e541e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/544a12c101f946d9a5d1310bfac0fadc] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=168.6 K 2024-11-12T07:49:17,561 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:17,561 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/6cf06cea2410475d964aa11f230f8cac, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ef9062e266d5429d81506c2ad29a9637, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/39ce90d289e246e4bbf98e8083f3c211, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ee1541bed3074a7f8a3086175b7e541e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/544a12c101f946d9a5d1310bfac0fadc] 2024-11-12T07:49:17,561 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting fe69f39511cf4c00887d4d0e68a2f3dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731397749261 2024-11-12T07:49:17,561 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cf06cea2410475d964aa11f230f8cac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731397749261 2024-11-12T07:49:17,561 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5908ccbbe40b436a96842b7bd9e8e170, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731397752703 2024-11-12T07:49:17,561 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef9062e266d5429d81506c2ad29a9637, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731397752697 2024-11-12T07:49:17,561 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd781bddca041228a9379c006cc58fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731397753379 2024-11-12T07:49:17,561 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39ce90d289e246e4bbf98e8083f3c211, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731397753379 2024-11-12T07:49:17,562 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a6e46d8bd34db0ad72678a56ff45dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731397755576 2024-11-12T07:49:17,562 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee1541bed3074a7f8a3086175b7e541e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731397755576 2024-11-12T07:49:17,562 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 3558e7e9f78e41f19320e09013afa4f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731397756765 2024-11-12T07:49:17,562 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 544a12c101f946d9a5d1310bfac0fadc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731397756752 2024-11-12T07:49:17,568 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:17,572 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#B#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:17,573 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/ef74fbff5ea647b6820ef15ac3445a67 is 50, key is test_row_0/B:col10/1731397757092/Put/seqid=0 2024-11-12T07:49:17,575 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112d1a939dbe0f2481f8d8cb3903f4b795e_d56ec85ac1d426368f9bcec18d0d9e10 store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:17,577 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112d1a939dbe0f2481f8d8cb3903f4b795e_d56ec85ac1d426368f9bcec18d0d9e10, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:17,577 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d1a939dbe0f2481f8d8cb3903f4b795e_d56ec85ac1d426368f9bcec18d0d9e10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:17,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742190_1366 (size=12527) 2024-11-12T07:49:17,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742191_1367 (size=4469) 2024-11-12T07:49:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T07:49:17,591 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#A#compaction#309 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:17,591 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/3253e2fc26b9431e8cdffea34be9d4be is 175, key is test_row_0/A:col10/1731397757092/Put/seqid=0 2024-11-12T07:49:17,595 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/ef74fbff5ea647b6820ef15ac3445a67 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/ef74fbff5ea647b6820ef15ac3445a67 2024-11-12T07:49:17,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742192_1368 (size=31481) 2024-11-12T07:49:17,606 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/B of d56ec85ac1d426368f9bcec18d0d9e10 into ef74fbff5ea647b6820ef15ac3445a67(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:17,606 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:17,606 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/B, priority=11, startTime=1731397757559; duration=0sec 2024-11-12T07:49:17,606 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:17,606 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:B 2024-11-12T07:49:17,606 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-12T07:49:17,607 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-12T07:49:17,607 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/C is initiating minor compaction (all files) 2024-11-12T07:49:17,607 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/C in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:17,607 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/acc00eea611645c59a6e781bc2b1791d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/57f90bf579ce4265bdcfda2470537687, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e58189e7dcd448909fe7b9ab3a63d049, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c6367defe8284936b6004912c21d63a9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/816a51ed90d0436988b9a46178324522] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=59.1 K 2024-11-12T07:49:17,607 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting acc00eea611645c59a6e781bc2b1791d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731397749261 2024-11-12T07:49:17,607 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 57f90bf579ce4265bdcfda2470537687, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731397752703 2024-11-12T07:49:17,607 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e58189e7dcd448909fe7b9ab3a63d049, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731397753379 2024-11-12T07:49:17,608 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c6367defe8284936b6004912c21d63a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731397755576 2024-11-12T07:49:17,608 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 816a51ed90d0436988b9a46178324522, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731397756765 2024-11-12T07:49:17,619 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:17,620 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/efb9d2cbfa27465d853275af7737c9ad is 50, key is test_row_0/C:col10/1731397757092/Put/seqid=0 2024-11-12T07:49:17,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742193_1369 (size=12527) 2024-11-12T07:49:17,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-12T07:49:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:17,633 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T07:49:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:17,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:17,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:17,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ee03be3242b24570b9eb78419b7af6e5_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397757165/Put/seqid=0 2024-11-12T07:49:17,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742194_1370 (size=12304) 2024-11-12T07:49:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T07:49:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:17,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:17,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397817843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397817844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397817844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397817958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397817958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397817958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:18,008 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/3253e2fc26b9431e8cdffea34be9d4be as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3253e2fc26b9431e8cdffea34be9d4be 2024-11-12T07:49:18,012 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/A of d56ec85ac1d426368f9bcec18d0d9e10 into 3253e2fc26b9431e8cdffea34be9d4be(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:18,012 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:18,012 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/A, priority=11, startTime=1731397757559; duration=0sec 2024-11-12T07:49:18,012 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:18,012 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:A 2024-11-12T07:49:18,030 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/efb9d2cbfa27465d853275af7737c9ad as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/efb9d2cbfa27465d853275af7737c9ad 2024-11-12T07:49:18,033 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/C of d56ec85ac1d426368f9bcec18d0d9e10 into efb9d2cbfa27465d853275af7737c9ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:18,033 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:18,033 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/C, priority=11, startTime=1731397757559; duration=0sec 2024-11-12T07:49:18,033 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:18,034 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:C 2024-11-12T07:49:18,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:18,047 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ee03be3242b24570b9eb78419b7af6e5_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee03be3242b24570b9eb78419b7af6e5_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/b06169c92a8a4049a30153d3de477eee, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:18,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/b06169c92a8a4049a30153d3de477eee is 175, key is test_row_0/A:col10/1731397757165/Put/seqid=0 2024-11-12T07:49:18,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742195_1371 (size=31105) 2024-11-12T07:49:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T07:49:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397818168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:18,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397818178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397818178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:18,451 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/b06169c92a8a4049a30153d3de477eee 2024-11-12T07:49:18,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:18,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397818478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:18,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397818488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:18,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397818488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:18,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/0d908110776d4c09a04588796ddda94a is 50, key is test_row_0/B:col10/1731397757165/Put/seqid=0 2024-11-12T07:49:18,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742196_1372 (size=12151) 2024-11-12T07:49:18,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T07:49:18,897 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/0d908110776d4c09a04588796ddda94a 2024-11-12T07:49:18,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/ceb6a855dac645ab90a7310c48ca73c6 is 50, key is test_row_0/C:col10/1731397757165/Put/seqid=0 2024-11-12T07:49:18,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742197_1373 (size=12151) 2024-11-12T07:49:18,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397818988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:18,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397818998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:19,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397819007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:19,308 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/ceb6a855dac645ab90a7310c48ca73c6 2024-11-12T07:49:19,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/b06169c92a8a4049a30153d3de477eee as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/b06169c92a8a4049a30153d3de477eee 2024-11-12T07:49:19,323 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/b06169c92a8a4049a30153d3de477eee, entries=150, sequenceid=173, filesize=30.4 K 2024-11-12T07:49:19,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/0d908110776d4c09a04588796ddda94a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/0d908110776d4c09a04588796ddda94a 2024-11-12T07:49:19,327 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/0d908110776d4c09a04588796ddda94a, entries=150, sequenceid=173, filesize=11.9 K 2024-11-12T07:49:19,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/ceb6a855dac645ab90a7310c48ca73c6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/ceb6a855dac645ab90a7310c48ca73c6 2024-11-12T07:49:19,332 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/ceb6a855dac645ab90a7310c48ca73c6, entries=150, sequenceid=173, filesize=11.9 K 2024-11-12T07:49:19,332 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d56ec85ac1d426368f9bcec18d0d9e10 in 1699ms, sequenceid=173, compaction requested=false 2024-11-12T07:49:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:19,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:19,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-12T07:49:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-12T07:49:19,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-12T07:49:19,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-11-12T07:49:19,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8570 sec 2024-11-12T07:49:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T07:49:19,628 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:19,628 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:19,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-12T07:49:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T07:49:19,630 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:19,630 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:19,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T07:49:19,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-12T07:49:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:19,783 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T07:49:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:19,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a8cdd6478e974a71addca0fbcd7eae47_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397757842/Put/seqid=0 2024-11-12T07:49:19,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742198_1374 (size=12304) 2024-11-12T07:49:19,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:19,818 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a8cdd6478e974a71addca0fbcd7eae47_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a8cdd6478e974a71addca0fbcd7eae47_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/e89be0ccb1f54d5e9f003d3db8247b64, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:19,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/e89be0ccb1f54d5e9f003d3db8247b64 is 175, key is test_row_0/A:col10/1731397757842/Put/seqid=0 2024-11-12T07:49:19,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742199_1375 (size=31105) 2024-11-12T07:49:19,831 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/e89be0ccb1f54d5e9f003d3db8247b64 2024-11-12T07:49:19,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/7fadd2f12ee641cbaa712cff72e41425 is 50, key is test_row_0/B:col10/1731397757842/Put/seqid=0 2024-11-12T07:49:19,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742200_1376 (size=12151) 2024-11-12T07:49:19,872 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/7fadd2f12ee641cbaa712cff72e41425 2024-11-12T07:49:19,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/7efdd26d6d664b519422a93738042941 is 50, key is test_row_0/C:col10/1731397757842/Put/seqid=0 2024-11-12T07:49:19,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742201_1377 (size=12151) 2024-11-12T07:49:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T07:49:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:19,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397820043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397820044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397820046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397820158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397820158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397820158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T07:49:20,299 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/7efdd26d6d664b519422a93738042941 2024-11-12T07:49:20,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/e89be0ccb1f54d5e9f003d3db8247b64 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e89be0ccb1f54d5e9f003d3db8247b64 2024-11-12T07:49:20,305 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e89be0ccb1f54d5e9f003d3db8247b64, entries=150, sequenceid=188, filesize=30.4 K 2024-11-12T07:49:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/7fadd2f12ee641cbaa712cff72e41425 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/7fadd2f12ee641cbaa712cff72e41425 2024-11-12T07:49:20,309 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/7fadd2f12ee641cbaa712cff72e41425, entries=150, sequenceid=188, filesize=11.9 K 2024-11-12T07:49:20,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/7efdd26d6d664b519422a93738042941 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/7efdd26d6d664b519422a93738042941 2024-11-12T07:49:20,313 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/7efdd26d6d664b519422a93738042941, entries=150, sequenceid=188, filesize=11.9 K 2024-11-12T07:49:20,313 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d56ec85ac1d426368f9bcec18d0d9e10 in 531ms, sequenceid=188, compaction requested=true 2024-11-12T07:49:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:20,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-12T07:49:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-12T07:49:20,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-12T07:49:20,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 685 msec 2024-11-12T07:49:20,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 689 msec 2024-11-12T07:49:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:20,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T07:49:20,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:20,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:20,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:20,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:20,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:20,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:20,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112021a93b3f8064295bad3f422b83a1f27_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397760375/Put/seqid=0 2024-11-12T07:49:20,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742202_1378 (size=12304) 2024-11-12T07:49:20,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397820403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397820423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397820423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397820527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397820548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397820548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397820738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397820758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397820758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-12T07:49:20,768 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:20,769 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-12T07:49:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T07:49:20,770 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:20,770 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:20,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:20,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:20,791 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112021a93b3f8064295bad3f422b83a1f27_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112021a93b3f8064295bad3f422b83a1f27_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:20,792 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/3f020be2aeda49dd8a7f0da6a425cef4, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:20,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/3f020be2aeda49dd8a7f0da6a425cef4 is 175, key is test_row_0/A:col10/1731397760375/Put/seqid=0 2024-11-12T07:49:20,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742203_1379 (size=31105) 2024-11-12T07:49:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T07:49:20,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:20,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:20,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:20,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:20,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:20,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397821057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:21,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397821078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397821078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T07:49:21,203 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/3f020be2aeda49dd8a7f0da6a425cef4 2024-11-12T07:49:21,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/b273a562adab4c67b1897e9f76c6defc is 50, key is test_row_0/B:col10/1731397760375/Put/seqid=0 2024-11-12T07:49:21,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742204_1380 (size=12151) 2024-11-12T07:49:21,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:21,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:21,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T07:49:21,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397821578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:21,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397821588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:21,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397821588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:21,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/b273a562adab4c67b1897e9f76c6defc 2024-11-12T07:49:21,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/8cc65cc5879d4ed6b88c1edd82bf87cc is 50, key is test_row_0/C:col10/1731397760375/Put/seqid=0 2024-11-12T07:49:21,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742205_1381 (size=12151) 2024-11-12T07:49:21,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:21,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:21,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:21,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T07:49:21,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:21,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:22,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/8cc65cc5879d4ed6b88c1edd82bf87cc 2024-11-12T07:49:22,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/3f020be2aeda49dd8a7f0da6a425cef4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3f020be2aeda49dd8a7f0da6a425cef4 2024-11-12T07:49:22,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3f020be2aeda49dd8a7f0da6a425cef4, entries=150, sequenceid=209, filesize=30.4 K 2024-11-12T07:49:22,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/b273a562adab4c67b1897e9f76c6defc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/b273a562adab4c67b1897e9f76c6defc 2024-11-12T07:49:22,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/b273a562adab4c67b1897e9f76c6defc, entries=150, sequenceid=209, filesize=11.9 K 2024-11-12T07:49:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/8cc65cc5879d4ed6b88c1edd82bf87cc as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/8cc65cc5879d4ed6b88c1edd82bf87cc 2024-11-12T07:49:22,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/8cc65cc5879d4ed6b88c1edd82bf87cc, entries=150, sequenceid=209, filesize=11.9 K 2024-11-12T07:49:22,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for d56ec85ac1d426368f9bcec18d0d9e10 in 1673ms, sequenceid=209, compaction requested=true 2024-11-12T07:49:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:22,050 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:49:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:22,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T07:49:22,050 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:49:22,053 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:49:22,053 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124796 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:49:22,053 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/A is initiating minor compaction (all files) 2024-11-12T07:49:22,053 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/B is initiating minor compaction (all files) 2024-11-12T07:49:22,053 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/A in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:22,053 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/B in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:22,053 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3253e2fc26b9431e8cdffea34be9d4be, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/b06169c92a8a4049a30153d3de477eee, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e89be0ccb1f54d5e9f003d3db8247b64, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3f020be2aeda49dd8a7f0da6a425cef4] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=121.9 K 2024-11-12T07:49:22,053 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/ef74fbff5ea647b6820ef15ac3445a67, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/0d908110776d4c09a04588796ddda94a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/7fadd2f12ee641cbaa712cff72e41425, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/b273a562adab4c67b1897e9f76c6defc] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=47.8 K 2024-11-12T07:49:22,053 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:22,053 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3253e2fc26b9431e8cdffea34be9d4be, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/b06169c92a8a4049a30153d3de477eee, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e89be0ccb1f54d5e9f003d3db8247b64, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3f020be2aeda49dd8a7f0da6a425cef4] 2024-11-12T07:49:22,053 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ef74fbff5ea647b6820ef15ac3445a67, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731397756765 2024-11-12T07:49:22,054 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3253e2fc26b9431e8cdffea34be9d4be, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731397756765 2024-11-12T07:49:22,054 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d908110776d4c09a04588796ddda94a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731397757158 2024-11-12T07:49:22,054 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting b06169c92a8a4049a30153d3de477eee, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731397757158 2024-11-12T07:49:22,054 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fadd2f12ee641cbaa712cff72e41425, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731397757838 2024-11-12T07:49:22,054 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e89be0ccb1f54d5e9f003d3db8247b64, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731397757838 2024-11-12T07:49:22,054 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting b273a562adab4c67b1897e9f76c6defc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731397760026 2024-11-12T07:49:22,054 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f020be2aeda49dd8a7f0da6a425cef4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731397760026 2024-11-12T07:49:22,059 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:22,066 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411124839ad6729f2434a8b3bf50029ced7b0_d56ec85ac1d426368f9bcec18d0d9e10 store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:22,067 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:22,068 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/687fc501177743efb24037cc20746562 is 50, key is test_row_0/B:col10/1731397760375/Put/seqid=0 2024-11-12T07:49:22,069 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411124839ad6729f2434a8b3bf50029ced7b0_d56ec85ac1d426368f9bcec18d0d9e10, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:22,069 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124839ad6729f2434a8b3bf50029ced7b0_d56ec85ac1d426368f9bcec18d0d9e10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:22,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742206_1382 (size=12663) 2024-11-12T07:49:22,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742207_1383 (size=4469) 2024-11-12T07:49:22,082 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/687fc501177743efb24037cc20746562 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/687fc501177743efb24037cc20746562 2024-11-12T07:49:22,082 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#A#compaction#321 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:22,082 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/624422d2a93046da91c2b6e55f0d2859 is 175, key is test_row_0/A:col10/1731397760375/Put/seqid=0 2024-11-12T07:49:22,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742208_1384 (size=31617) 2024-11-12T07:49:22,088 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/B of d56ec85ac1d426368f9bcec18d0d9e10 into 687fc501177743efb24037cc20746562(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:22,088 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:22,088 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/B, priority=12, startTime=1731397762050; duration=0sec 2024-11-12T07:49:22,088 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:22,089 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:B 2024-11-12T07:49:22,089 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:49:22,090 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:49:22,090 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/C is initiating minor compaction (all files) 2024-11-12T07:49:22,090 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/C in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:22,090 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/efb9d2cbfa27465d853275af7737c9ad, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/ceb6a855dac645ab90a7310c48ca73c6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/7efdd26d6d664b519422a93738042941, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/8cc65cc5879d4ed6b88c1edd82bf87cc] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=47.8 K 2024-11-12T07:49:22,091 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting efb9d2cbfa27465d853275af7737c9ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731397756765 2024-11-12T07:49:22,091 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb6a855dac645ab90a7310c48ca73c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731397757158 2024-11-12T07:49:22,091 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 7efdd26d6d664b519422a93738042941, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731397757838 2024-11-12T07:49:22,091 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc65cc5879d4ed6b88c1edd82bf87cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731397760026 2024-11-12T07:49:22,094 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/624422d2a93046da91c2b6e55f0d2859 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/624422d2a93046da91c2b6e55f0d2859 2024-11-12T07:49:22,097 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/A of d56ec85ac1d426368f9bcec18d0d9e10 into 624422d2a93046da91c2b6e55f0d2859(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:22,098 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:22,098 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/A, priority=12, startTime=1731397762050; duration=0sec 2024-11-12T07:49:22,098 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:22,098 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:A 2024-11-12T07:49:22,099 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:22,099 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/d7694d2e24eb48afab41974572f4a2fb is 50, key is test_row_0/C:col10/1731397760375/Put/seqid=0 2024-11-12T07:49:22,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742209_1385 (size=12663) 2024-11-12T07:49:22,110 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/d7694d2e24eb48afab41974572f4a2fb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/d7694d2e24eb48afab41974572f4a2fb 2024-11-12T07:49:22,114 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/C of d56ec85ac1d426368f9bcec18d0d9e10 into d7694d2e24eb48afab41974572f4a2fb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:22,114 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:22,114 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/C, priority=12, startTime=1731397762050; duration=0sec 2024-11-12T07:49:22,114 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:22,114 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:C 2024-11-12T07:49:22,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-12T07:49:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:22,144 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:49:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112634cf8e7f9a34a178cb3389345dc3868_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397760410/Put/seqid=0 2024-11-12T07:49:22,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742210_1386 (size=12304) 2024-11-12T07:49:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:22,557 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112634cf8e7f9a34a178cb3389345dc3868_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112634cf8e7f9a34a178cb3389345dc3868_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:22,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/45254d3b220e45c681ad5d16c82900e4, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:22,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/45254d3b220e45c681ad5d16c82900e4 is 175, key is test_row_0/A:col10/1731397760410/Put/seqid=0 2024-11-12T07:49:22,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742211_1387 (size=31105) 2024-11-12T07:49:22,566 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/45254d3b220e45c681ad5d16c82900e4 2024-11-12T07:49:22,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/49e76037db1c42408adbacf0cc5a4365 is 50, key is test_row_0/B:col10/1731397760410/Put/seqid=0 2024-11-12T07:49:22,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742212_1388 (size=12151) 2024-11-12T07:49:22,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397822646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397822652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397822653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397822758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397822768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397822768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T07:49:22,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397822968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397822978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397822978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:22,989 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/49e76037db1c42408adbacf0cc5a4365 2024-11-12T07:49:22,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/6983acf943444abda0f55fbf141eb524 is 50, key is test_row_0/C:col10/1731397760410/Put/seqid=0 2024-11-12T07:49:22,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742213_1389 (size=12151) 2024-11-12T07:49:23,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397823277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:23,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397823287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397823288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:23,400 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/6983acf943444abda0f55fbf141eb524 2024-11-12T07:49:23,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/45254d3b220e45c681ad5d16c82900e4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/45254d3b220e45c681ad5d16c82900e4 2024-11-12T07:49:23,406 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/45254d3b220e45c681ad5d16c82900e4, entries=150, sequenceid=224, filesize=30.4 K 2024-11-12T07:49:23,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/49e76037db1c42408adbacf0cc5a4365 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/49e76037db1c42408adbacf0cc5a4365 2024-11-12T07:49:23,410 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/49e76037db1c42408adbacf0cc5a4365, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T07:49:23,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/6983acf943444abda0f55fbf141eb524 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6983acf943444abda0f55fbf141eb524 2024-11-12T07:49:23,419 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6983acf943444abda0f55fbf141eb524, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T07:49:23,419 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d56ec85ac1d426368f9bcec18d0d9e10 in 1275ms, sequenceid=224, compaction requested=false 2024-11-12T07:49:23,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:23,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:23,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-12T07:49:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-12T07:49:23,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-12T07:49:23,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6510 sec 2024-11-12T07:49:23,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.6540 sec 2024-11-12T07:49:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:23,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T07:49:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:23,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:23,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:23,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397823819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:23,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397823820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:23,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397823821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:23,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ee93d1d0a64d4d11b64fc1da27f4e2a6_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397762651/Put/seqid=0 2024-11-12T07:49:23,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742214_1390 (size=14794) 2024-11-12T07:49:23,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397823928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:23,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397823938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397823938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397824138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397824158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:24,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397824158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56652 deadline: 1731397824177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:24,183 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in d56ec85ac1d426368f9bcec18d0d9e10 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56636 deadline: 1731397824218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:24,224 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in d56ec85ac1d426368f9bcec18d0d9e10 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18325 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,240 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:24,243 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ee93d1d0a64d4d11b64fc1da27f4e2a6_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee93d1d0a64d4d11b64fc1da27f4e2a6_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:24,244 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/7698f6b591fb4f4287435f73e0fc41f4, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:24,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/7698f6b591fb4f4287435f73e0fc41f4 is 175, key is test_row_0/A:col10/1731397762651/Put/seqid=0 2024-11-12T07:49:24,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742215_1391 (size=39749) 2024-11-12T07:49:24,249 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/7698f6b591fb4f4287435f73e0fc41f4 2024-11-12T07:49:24,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/4e1d45c96dbb4f38b9ccb87eb35aca31 is 50, key is test_row_0/B:col10/1731397762651/Put/seqid=0 2024-11-12T07:49:24,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742216_1392 (size=12151) 2024-11-12T07:49:24,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397824448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397824468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:24,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397824468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:24,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:24,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/4e1d45c96dbb4f38b9ccb87eb35aca31 2024-11-12T07:49:24,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/23b37a10f5754d308ba08729bff7c664 is 50, key is test_row_0/C:col10/1731397762651/Put/seqid=0 2024-11-12T07:49:24,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742217_1393 (size=12151) 2024-11-12T07:49:24,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/23b37a10f5754d308ba08729bff7c664 2024-11-12T07:49:24,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/7698f6b591fb4f4287435f73e0fc41f4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/7698f6b591fb4f4287435f73e0fc41f4 2024-11-12T07:49:24,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/7698f6b591fb4f4287435f73e0fc41f4, entries=200, sequenceid=245, filesize=38.8 K 2024-11-12T07:49:24,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/4e1d45c96dbb4f38b9ccb87eb35aca31 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4e1d45c96dbb4f38b9ccb87eb35aca31 2024-11-12T07:49:24,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4e1d45c96dbb4f38b9ccb87eb35aca31, entries=150, sequenceid=245, filesize=11.9 K 2024-11-12T07:49:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/23b37a10f5754d308ba08729bff7c664 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/23b37a10f5754d308ba08729bff7c664 2024-11-12T07:49:24,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/23b37a10f5754d308ba08729bff7c664, entries=150, sequenceid=245, filesize=11.9 K 2024-11-12T07:49:24,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d56ec85ac1d426368f9bcec18d0d9e10 in 909ms, sequenceid=245, compaction requested=true 2024-11-12T07:49:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:24,701 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:24,701 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d56ec85ac1d426368f9bcec18d0d9e10:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:24,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:24,702 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:24,702 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:24,702 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/A is initiating minor compaction (all files) 2024-11-12T07:49:24,702 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/B is initiating minor compaction (all files) 2024-11-12T07:49:24,702 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/B in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:24,702 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/A in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:24,702 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/624422d2a93046da91c2b6e55f0d2859, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/45254d3b220e45c681ad5d16c82900e4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/7698f6b591fb4f4287435f73e0fc41f4] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=100.1 K 2024-11-12T07:49:24,702 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/687fc501177743efb24037cc20746562, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/49e76037db1c42408adbacf0cc5a4365, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4e1d45c96dbb4f38b9ccb87eb35aca31] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=36.1 K 2024-11-12T07:49:24,702 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:24,702 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/624422d2a93046da91c2b6e55f0d2859, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/45254d3b220e45c681ad5d16c82900e4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/7698f6b591fb4f4287435f73e0fc41f4] 2024-11-12T07:49:24,702 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 687fc501177743efb24037cc20746562, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731397760026 2024-11-12T07:49:24,702 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624422d2a93046da91c2b6e55f0d2859, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731397760026 2024-11-12T07:49:24,702 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 49e76037db1c42408adbacf0cc5a4365, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731397760401 2024-11-12T07:49:24,703 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45254d3b220e45c681ad5d16c82900e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731397760401 2024-11-12T07:49:24,703 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e1d45c96dbb4f38b9ccb87eb35aca31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397762630 2024-11-12T07:49:24,703 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7698f6b591fb4f4287435f73e0fc41f4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397762630 2024-11-12T07:49:24,723 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:24,726 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#B#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:24,726 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/c68743ee1a2a43a4b760764752c5614c is 50, key is test_row_0/B:col10/1731397762651/Put/seqid=0 2024-11-12T07:49:24,728 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111200b1fb889dc048c4974685301fb65b0d_d56ec85ac1d426368f9bcec18d0d9e10 store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:24,730 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111200b1fb889dc048c4974685301fb65b0d_d56ec85ac1d426368f9bcec18d0d9e10, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:24,731 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111200b1fb889dc048c4974685301fb65b0d_d56ec85ac1d426368f9bcec18d0d9e10 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:24,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742218_1394 (size=12765) 2024-11-12T07:49:24,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742219_1395 (size=4469) 2024-11-12T07:49:24,735 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#A#compaction#330 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:24,736 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/27210f088e004999997b64b6c9aa128b is 175, key is test_row_0/A:col10/1731397762651/Put/seqid=0 2024-11-12T07:49:24,739 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/c68743ee1a2a43a4b760764752c5614c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/c68743ee1a2a43a4b760764752c5614c 2024-11-12T07:49:24,743 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/B of d56ec85ac1d426368f9bcec18d0d9e10 into c68743ee1a2a43a4b760764752c5614c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:24,743 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:24,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742220_1396 (size=31719) 2024-11-12T07:49:24,743 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/B, priority=13, startTime=1731397764701; duration=0sec 2024-11-12T07:49:24,743 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:24,743 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:B 2024-11-12T07:49:24,744 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:24,744 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:24,744 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): d56ec85ac1d426368f9bcec18d0d9e10/C is initiating minor compaction (all files) 2024-11-12T07:49:24,744 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d56ec85ac1d426368f9bcec18d0d9e10/C in TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:24,744 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/d7694d2e24eb48afab41974572f4a2fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6983acf943444abda0f55fbf141eb524, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/23b37a10f5754d308ba08729bff7c664] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp, totalSize=36.1 K 2024-11-12T07:49:24,745 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d7694d2e24eb48afab41974572f4a2fb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731397760026 2024-11-12T07:49:24,745 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 6983acf943444abda0f55fbf141eb524, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731397760401 2024-11-12T07:49:24,745 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b37a10f5754d308ba08729bff7c664, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731397762630 2024-11-12T07:49:24,753 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d56ec85ac1d426368f9bcec18d0d9e10#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:24,753 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/559c766a7eed42ef8fafca8af4e9c3ff is 50, key is test_row_0/C:col10/1731397762651/Put/seqid=0 2024-11-12T07:49:24,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742221_1397 (size=12765) 2024-11-12T07:49:24,766 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/559c766a7eed42ef8fafca8af4e9c3ff as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/559c766a7eed42ef8fafca8af4e9c3ff 2024-11-12T07:49:24,772 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/C of d56ec85ac1d426368f9bcec18d0d9e10 into 559c766a7eed42ef8fafca8af4e9c3ff(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:24,772 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:24,772 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/C, priority=13, startTime=1731397764701; duration=0sec 2024-11-12T07:49:24,772 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:24,772 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:C 2024-11-12T07:49:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T07:49:24,938 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:24,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-12T07:49:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T07:49:24,940 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:24,941 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:24,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:24,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:49:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:24,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111293b1b7efe68a4a13a62b680a760bacb4_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397763817/Put/seqid=0 2024-11-12T07:49:24,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742222_1398 (size=14844) 2024-11-12T07:49:24,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:24,984 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111293b1b7efe68a4a13a62b680a760bacb4_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293b1b7efe68a4a13a62b680a760bacb4_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:24,985 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/4af16289b8fa4ca5a8b633dd26cbbba0, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:24,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/4af16289b8fa4ca5a8b633dd26cbbba0 is 175, key is test_row_0/A:col10/1731397763817/Put/seqid=0 2024-11-12T07:49:24,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742223_1399 (size=39799) 2024-11-12T07:49:24,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/4af16289b8fa4ca5a8b633dd26cbbba0 2024-11-12T07:49:25,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/bb5a3abb258f437a8a68f2ee733b478a is 50, key is test_row_0/B:col10/1731397763817/Put/seqid=0 2024-11-12T07:49:25,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742224_1400 (size=12201) 2024-11-12T07:49:25,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397825015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397825019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397825021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T07:49:25,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T07:49:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397825137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397825137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397825138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,148 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/27210f088e004999997b64b6c9aa128b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/27210f088e004999997b64b6c9aa128b 2024-11-12T07:49:25,151 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d56ec85ac1d426368f9bcec18d0d9e10/A of d56ec85ac1d426368f9bcec18d0d9e10 into 27210f088e004999997b64b6c9aa128b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:25,151 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:25,151 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., storeName=d56ec85ac1d426368f9bcec18d0d9e10/A, priority=13, startTime=1731397764701; duration=0sec 2024-11-12T07:49:25,152 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:25,152 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d56ec85ac1d426368f9bcec18d0d9e10:A 2024-11-12T07:49:25,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T07:49:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T07:49:25,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397825347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397825358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397825358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T07:49:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/bb5a3abb258f437a8a68f2ee733b478a 2024-11-12T07:49:25,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/02e3a46c60654ab9a0295dbf6d426ac5 is 50, key is test_row_0/C:col10/1731397763817/Put/seqid=0 2024-11-12T07:49:25,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742225_1401 (size=12201) 2024-11-12T07:49:25,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T07:49:25,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:25,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T07:49:25,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56668 deadline: 1731397825657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56676 deadline: 1731397825668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56674 deadline: 1731397825668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:25,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:49:25,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T07:49:25,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:25,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:25,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/02e3a46c60654ab9a0295dbf6d426ac5 2024-11-12T07:49:25,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/4af16289b8fa4ca5a8b633dd26cbbba0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/4af16289b8fa4ca5a8b633dd26cbbba0 2024-11-12T07:49:25,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/4af16289b8fa4ca5a8b633dd26cbbba0, entries=200, sequenceid=259, filesize=38.9 K 2024-11-12T07:49:25,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/bb5a3abb258f437a8a68f2ee733b478a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bb5a3abb258f437a8a68f2ee733b478a 2024-11-12T07:49:25,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bb5a3abb258f437a8a68f2ee733b478a, entries=150, sequenceid=259, filesize=11.9 K 2024-11-12T07:49:25,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/02e3a46c60654ab9a0295dbf6d426ac5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/02e3a46c60654ab9a0295dbf6d426ac5 2024-11-12T07:49:25,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/02e3a46c60654ab9a0295dbf6d426ac5, entries=150, sequenceid=259, filesize=11.9 K 2024-11-12T07:49:25,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d56ec85ac1d426368f9bcec18d0d9e10 in 890ms, sequenceid=259, compaction requested=false 2024-11-12T07:49:25,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:25,850 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-11-12T07:49:25,850 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:25,850 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,850 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:25,853 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-11-12T07:49:25,853 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:25,853 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,853 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:25,854 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-11-12T07:49:25,854 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:25,854 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,854 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:25,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-12T07:49:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:25,857 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T07:49:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:25,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:25,858 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-11-12T07:49:25,858 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:25,858 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,858 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:25,859 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-11-12T07:49:25,859 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:25,859 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,859 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:25,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:25,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b1167853bdfc4a59871b334cc9adc0ed_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397765018/Put/seqid=0 2024-11-12T07:49:25,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742226_1402 (size=12454) 2024-11-12T07:49:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T07:49:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:26,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. as already flushing 2024-11-12T07:49:26,183 INFO [Thread-1472 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1472. 2024-11-12T07:49:26,183 DEBUG [Thread-1472 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:26,184 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:26,184 DEBUG [Thread-1472 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:26,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:26,192 INFO [Thread-1474 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1474. 2024-11-12T07:49:26,192 INFO [Thread-1476 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1476. 2024-11-12T07:49:26,192 DEBUG [Thread-1474 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:26,192 DEBUG [Thread-1476 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:26,192 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:26,192 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:26,192 DEBUG [Thread-1474 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:26,192 DEBUG [Thread-1476 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:26,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:26,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:26,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:26,279 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b1167853bdfc4a59871b334cc9adc0ed_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b1167853bdfc4a59871b334cc9adc0ed_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:26,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/eeb92b9f07b7419bad594ef752162f0a, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:26,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/eeb92b9f07b7419bad594ef752162f0a is 175, key is test_row_0/A:col10/1731397765018/Put/seqid=0 2024-11-12T07:49:26,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742227_1403 (size=31255) 2024-11-12T07:49:26,686 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/eeb92b9f07b7419bad594ef752162f0a 2024-11-12T07:49:26,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/23fd8cb5f9814065a4b67b576681b89b is 50, key is test_row_0/B:col10/1731397765018/Put/seqid=0 2024-11-12T07:49:26,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742228_1404 (size=12301) 2024-11-12T07:49:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T07:49:27,106 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/23fd8cb5f9814065a4b67b576681b89b 2024-11-12T07:49:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/b0cb4ef0983f46e78b98d90a7a8d4d2d is 50, key is test_row_0/C:col10/1731397765018/Put/seqid=0 2024-11-12T07:49:27,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742229_1405 (size=12301) 2024-11-12T07:49:27,525 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/b0cb4ef0983f46e78b98d90a7a8d4d2d 2024-11-12T07:49:27,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/eeb92b9f07b7419bad594ef752162f0a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/eeb92b9f07b7419bad594ef752162f0a 2024-11-12T07:49:27,539 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/eeb92b9f07b7419bad594ef752162f0a, entries=150, sequenceid=280, filesize=30.5 K 2024-11-12T07:49:27,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/23fd8cb5f9814065a4b67b576681b89b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/23fd8cb5f9814065a4b67b576681b89b 2024-11-12T07:49:27,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/23fd8cb5f9814065a4b67b576681b89b, entries=150, sequenceid=280, filesize=12.0 K 2024-11-12T07:49:27,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/b0cb4ef0983f46e78b98d90a7a8d4d2d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/b0cb4ef0983f46e78b98d90a7a8d4d2d 2024-11-12T07:49:27,548 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/b0cb4ef0983f46e78b98d90a7a8d4d2d, entries=150, sequenceid=280, filesize=12.0 K 2024-11-12T07:49:27,549 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for d56ec85ac1d426368f9bcec18d0d9e10 in 1692ms, sequenceid=280, compaction requested=true 2024-11-12T07:49:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d56ec85ac1d426368f9bcec18d0d9e10: 2024-11-12T07:49:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-12T07:49:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-12T07:49:27,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-12T07:49:27,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6090 sec 2024-11-12T07:49:27,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.6150 sec 2024-11-12T07:49:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-12T07:49:29,110 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:29,129 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:49:33,866 WARN [regionserver/f9fc96254ef8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-12T07:49:33,869 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e8089dcb0d654356ad69728776022c75, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/0671c41696b944b59abf78f902f17364, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/04ece208b2894f52b8c5de4a60a79833, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3695f4bdb29d4800afc702fa2ed1eac9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ecb36764c93d4e4c838af82417fc3195, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/6cf06cea2410475d964aa11f230f8cac, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c8a6a6c93556494e9cbc80555079ea8a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ef9062e266d5429d81506c2ad29a9637, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/39ce90d289e246e4bbf98e8083f3c211, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ee1541bed3074a7f8a3086175b7e541e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/544a12c101f946d9a5d1310bfac0fadc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3253e2fc26b9431e8cdffea34be9d4be, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/b06169c92a8a4049a30153d3de477eee, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e89be0ccb1f54d5e9f003d3db8247b64, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/624422d2a93046da91c2b6e55f0d2859, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3f020be2aeda49dd8a7f0da6a425cef4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/45254d3b220e45c681ad5d16c82900e4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/7698f6b591fb4f4287435f73e0fc41f4] to archive 2024-11-12T07:49:33,870 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:49:33,872 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e8089dcb0d654356ad69728776022c75 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e8089dcb0d654356ad69728776022c75 2024-11-12T07:49:33,873 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/0671c41696b944b59abf78f902f17364 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/0671c41696b944b59abf78f902f17364 2024-11-12T07:49:33,875 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/04ece208b2894f52b8c5de4a60a79833 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/04ece208b2894f52b8c5de4a60a79833 2024-11-12T07:49:33,876 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3695f4bdb29d4800afc702fa2ed1eac9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3695f4bdb29d4800afc702fa2ed1eac9 2024-11-12T07:49:33,877 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ecb36764c93d4e4c838af82417fc3195 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ecb36764c93d4e4c838af82417fc3195 2024-11-12T07:49:33,878 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/6cf06cea2410475d964aa11f230f8cac to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/6cf06cea2410475d964aa11f230f8cac 2024-11-12T07:49:33,879 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c8a6a6c93556494e9cbc80555079ea8a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c8a6a6c93556494e9cbc80555079ea8a 2024-11-12T07:49:33,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ef9062e266d5429d81506c2ad29a9637 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ef9062e266d5429d81506c2ad29a9637 2024-11-12T07:49:33,882 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/39ce90d289e246e4bbf98e8083f3c211 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/39ce90d289e246e4bbf98e8083f3c211 2024-11-12T07:49:33,883 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ee1541bed3074a7f8a3086175b7e541e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/ee1541bed3074a7f8a3086175b7e541e 2024-11-12T07:49:33,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/544a12c101f946d9a5d1310bfac0fadc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/544a12c101f946d9a5d1310bfac0fadc 2024-11-12T07:49:33,885 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3253e2fc26b9431e8cdffea34be9d4be to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3253e2fc26b9431e8cdffea34be9d4be 2024-11-12T07:49:33,886 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/b06169c92a8a4049a30153d3de477eee to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/b06169c92a8a4049a30153d3de477eee 2024-11-12T07:49:33,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e89be0ccb1f54d5e9f003d3db8247b64 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/e89be0ccb1f54d5e9f003d3db8247b64 2024-11-12T07:49:33,888 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/624422d2a93046da91c2b6e55f0d2859 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/624422d2a93046da91c2b6e55f0d2859 2024-11-12T07:49:33,889 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3f020be2aeda49dd8a7f0da6a425cef4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/3f020be2aeda49dd8a7f0da6a425cef4 2024-11-12T07:49:33,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/45254d3b220e45c681ad5d16c82900e4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/45254d3b220e45c681ad5d16c82900e4 2024-11-12T07:49:33,892 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/7698f6b591fb4f4287435f73e0fc41f4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/7698f6b591fb4f4287435f73e0fc41f4 2024-11-12T07:49:33,895 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/86a4386297634e89a873e34393131c3e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/8439dea76ee14c33844fa63a53f56916, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/60e02c8e6b1f474c8540c781de8e88a3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5edafaadef8d47bdb83f001d9d531f59, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bab091cb300241b4ac8814846a7c7aee, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/fe69f39511cf4c00887d4d0e68a2f3dc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/543b5d65f570431c81051a5caa8aab77, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5908ccbbe40b436a96842b7bd9e8e170, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4bd781bddca041228a9379c006cc58fa, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/e1a6e46d8bd34db0ad72678a56ff45dc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/ef74fbff5ea647b6820ef15ac3445a67, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/3558e7e9f78e41f19320e09013afa4f4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/0d908110776d4c09a04588796ddda94a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/7fadd2f12ee641cbaa712cff72e41425, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/687fc501177743efb24037cc20746562, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/b273a562adab4c67b1897e9f76c6defc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/49e76037db1c42408adbacf0cc5a4365, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4e1d45c96dbb4f38b9ccb87eb35aca31] to archive 2024-11-12T07:49:33,896 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:49:33,898 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/86a4386297634e89a873e34393131c3e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/86a4386297634e89a873e34393131c3e 2024-11-12T07:49:33,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/8439dea76ee14c33844fa63a53f56916 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/8439dea76ee14c33844fa63a53f56916 2024-11-12T07:49:33,900 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/60e02c8e6b1f474c8540c781de8e88a3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/60e02c8e6b1f474c8540c781de8e88a3 2024-11-12T07:49:33,901 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5edafaadef8d47bdb83f001d9d531f59 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5edafaadef8d47bdb83f001d9d531f59 2024-11-12T07:49:33,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bab091cb300241b4ac8814846a7c7aee to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bab091cb300241b4ac8814846a7c7aee 2024-11-12T07:49:33,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/fe69f39511cf4c00887d4d0e68a2f3dc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/fe69f39511cf4c00887d4d0e68a2f3dc 2024-11-12T07:49:33,903 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/543b5d65f570431c81051a5caa8aab77 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/543b5d65f570431c81051a5caa8aab77 2024-11-12T07:49:33,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5908ccbbe40b436a96842b7bd9e8e170 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/5908ccbbe40b436a96842b7bd9e8e170 2024-11-12T07:49:33,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4bd781bddca041228a9379c006cc58fa to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4bd781bddca041228a9379c006cc58fa 2024-11-12T07:49:33,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/e1a6e46d8bd34db0ad72678a56ff45dc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/e1a6e46d8bd34db0ad72678a56ff45dc 2024-11-12T07:49:33,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/ef74fbff5ea647b6820ef15ac3445a67 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/ef74fbff5ea647b6820ef15ac3445a67 2024-11-12T07:49:33,907 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/3558e7e9f78e41f19320e09013afa4f4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/3558e7e9f78e41f19320e09013afa4f4 2024-11-12T07:49:33,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/0d908110776d4c09a04588796ddda94a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/0d908110776d4c09a04588796ddda94a 2024-11-12T07:49:33,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/7fadd2f12ee641cbaa712cff72e41425 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/7fadd2f12ee641cbaa712cff72e41425 2024-11-12T07:49:33,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/687fc501177743efb24037cc20746562 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/687fc501177743efb24037cc20746562 2024-11-12T07:49:33,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/b273a562adab4c67b1897e9f76c6defc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/b273a562adab4c67b1897e9f76c6defc 2024-11-12T07:49:33,912 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/49e76037db1c42408adbacf0cc5a4365 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/49e76037db1c42408adbacf0cc5a4365 2024-11-12T07:49:33,913 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4e1d45c96dbb4f38b9ccb87eb35aca31 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/4e1d45c96dbb4f38b9ccb87eb35aca31 2024-11-12T07:49:33,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5ada7618a904cffa02d1b83814ce920, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6dc149a1509648a0a0dd53fdb5c309e9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/38f63e0166714daea3cb1965464575b9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5758c63b6fb4f048c297279b497d330, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/64c433ee8076403a94f93202eb30f95e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/acc00eea611645c59a6e781bc2b1791d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e87d47da7e1f431db9e663fad37fa8da, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/57f90bf579ce4265bdcfda2470537687, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e58189e7dcd448909fe7b9ab3a63d049, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c6367defe8284936b6004912c21d63a9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/efb9d2cbfa27465d853275af7737c9ad, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/816a51ed90d0436988b9a46178324522, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/ceb6a855dac645ab90a7310c48ca73c6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/7efdd26d6d664b519422a93738042941, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/d7694d2e24eb48afab41974572f4a2fb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/8cc65cc5879d4ed6b88c1edd82bf87cc, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6983acf943444abda0f55fbf141eb524, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/23b37a10f5754d308ba08729bff7c664] to archive 2024-11-12T07:49:33,917 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:49:33,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5ada7618a904cffa02d1b83814ce920 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5ada7618a904cffa02d1b83814ce920 2024-11-12T07:49:33,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6dc149a1509648a0a0dd53fdb5c309e9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6dc149a1509648a0a0dd53fdb5c309e9 2024-11-12T07:49:33,921 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/38f63e0166714daea3cb1965464575b9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/38f63e0166714daea3cb1965464575b9 2024-11-12T07:49:33,922 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5758c63b6fb4f048c297279b497d330 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c5758c63b6fb4f048c297279b497d330 2024-11-12T07:49:33,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/64c433ee8076403a94f93202eb30f95e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/64c433ee8076403a94f93202eb30f95e 2024-11-12T07:49:33,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/acc00eea611645c59a6e781bc2b1791d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/acc00eea611645c59a6e781bc2b1791d 2024-11-12T07:49:33,924 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e87d47da7e1f431db9e663fad37fa8da to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e87d47da7e1f431db9e663fad37fa8da 2024-11-12T07:49:33,925 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/57f90bf579ce4265bdcfda2470537687 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/57f90bf579ce4265bdcfda2470537687 2024-11-12T07:49:33,926 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e58189e7dcd448909fe7b9ab3a63d049 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/e58189e7dcd448909fe7b9ab3a63d049 2024-11-12T07:49:33,927 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c6367defe8284936b6004912c21d63a9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/c6367defe8284936b6004912c21d63a9 2024-11-12T07:49:33,928 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/efb9d2cbfa27465d853275af7737c9ad to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/efb9d2cbfa27465d853275af7737c9ad 2024-11-12T07:49:33,928 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/816a51ed90d0436988b9a46178324522 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/816a51ed90d0436988b9a46178324522 2024-11-12T07:49:33,929 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/ceb6a855dac645ab90a7310c48ca73c6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/ceb6a855dac645ab90a7310c48ca73c6 2024-11-12T07:49:33,930 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/7efdd26d6d664b519422a93738042941 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/7efdd26d6d664b519422a93738042941 2024-11-12T07:49:33,931 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/d7694d2e24eb48afab41974572f4a2fb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/d7694d2e24eb48afab41974572f4a2fb 2024-11-12T07:49:33,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/8cc65cc5879d4ed6b88c1edd82bf87cc to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/8cc65cc5879d4ed6b88c1edd82bf87cc 2024-11-12T07:49:33,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6983acf943444abda0f55fbf141eb524 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/6983acf943444abda0f55fbf141eb524 2024-11-12T07:49:33,933 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f9fc96254ef8:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/23b37a10f5754d308ba08729bff7c664 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/23b37a10f5754d308ba08729bff7c664 2024-11-12T07:49:34,273 INFO [Thread-1478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1478. 2024-11-12T07:49:34,273 DEBUG [Thread-1478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:34,273 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:34,274 DEBUG [Thread-1478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:34,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:34,332 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-11-12T07:49:34,332 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:34,332 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:34,333 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:34,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1288 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3861 rows 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1290 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3863 rows 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1288 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3862 rows 2024-11-12T07:49:34,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1284 2024-11-12T07:49:34,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3841 rows 2024-11-12T07:49:34,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1286 2024-11-12T07:49:34,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3849 rows 2024-11-12T07:49:34,334 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:49:34,334 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:49:34,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:34,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:34,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:34,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T07:49:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T07:49:34,340 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397774340"}]},"ts":"1731397774340"} 2024-11-12T07:49:34,342 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T07:49:34,342 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T07:49:34,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:49:34,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, UNASSIGN}] 2024-11-12T07:49:34,345 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, UNASSIGN 2024-11-12T07:49:34,346 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=d56ec85ac1d426368f9bcec18d0d9e10, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:34,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, UNASSIGN because future has completed 2024-11-12T07:49:34,348 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:49:34,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:49:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T07:49:34,501 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:34,501 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:49:34,501 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing d56ec85ac1d426368f9bcec18d0d9e10, disabling compactions & flushes 2024-11-12T07:49:34,501 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:34,501 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:34,501 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. after waiting 0 ms 2024-11-12T07:49:34,501 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:34,501 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing d56ec85ac1d426368f9bcec18d0d9e10 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T07:49:34,502 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=A 2024-11-12T07:49:34,502 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:34,502 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=B 2024-11-12T07:49:34,502 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:34,502 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d56ec85ac1d426368f9bcec18d0d9e10, store=C 2024-11-12T07:49:34,502 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:34,510 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111272da0eda773d4f73bd788adeb2ccaab8_d56ec85ac1d426368f9bcec18d0d9e10 is 50, key is test_row_0/A:col10/1731397766189/Put/seqid=0 2024-11-12T07:49:34,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742230_1406 (size=12454) 2024-11-12T07:49:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T07:49:34,916 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:34,927 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111272da0eda773d4f73bd788adeb2ccaab8_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111272da0eda773d4f73bd788adeb2ccaab8_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:34,929 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/c9fb6d4fca274b2baaff8646ffa9329f, store: [table=TestAcidGuarantees family=A region=d56ec85ac1d426368f9bcec18d0d9e10] 2024-11-12T07:49:34,930 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/c9fb6d4fca274b2baaff8646ffa9329f is 175, key is test_row_0/A:col10/1731397766189/Put/seqid=0 2024-11-12T07:49:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742231_1407 (size=31255) 2024-11-12T07:49:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T07:49:35,334 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/c9fb6d4fca274b2baaff8646ffa9329f 2024-11-12T07:49:35,346 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/170af6a6b0ca4114a7bd97c26fe72ea9 is 50, key is test_row_0/B:col10/1731397766189/Put/seqid=0 2024-11-12T07:49:35,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742232_1408 (size=12301) 2024-11-12T07:49:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T07:49:35,751 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/170af6a6b0ca4114a7bd97c26fe72ea9 2024-11-12T07:49:35,762 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/aff628673fc24ad0a6cfc96ae7a0b765 is 50, key is test_row_0/C:col10/1731397766189/Put/seqid=0 2024-11-12T07:49:35,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742233_1409 (size=12301) 2024-11-12T07:49:36,168 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/aff628673fc24ad0a6cfc96ae7a0b765 2024-11-12T07:49:36,178 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/A/c9fb6d4fca274b2baaff8646ffa9329f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c9fb6d4fca274b2baaff8646ffa9329f 2024-11-12T07:49:36,182 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c9fb6d4fca274b2baaff8646ffa9329f, entries=150, sequenceid=288, filesize=30.5 K 2024-11-12T07:49:36,182 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/B/170af6a6b0ca4114a7bd97c26fe72ea9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/170af6a6b0ca4114a7bd97c26fe72ea9 2024-11-12T07:49:36,186 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/170af6a6b0ca4114a7bd97c26fe72ea9, entries=150, sequenceid=288, filesize=12.0 K 2024-11-12T07:49:36,186 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/.tmp/C/aff628673fc24ad0a6cfc96ae7a0b765 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/aff628673fc24ad0a6cfc96ae7a0b765 2024-11-12T07:49:36,189 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/aff628673fc24ad0a6cfc96ae7a0b765, entries=150, sequenceid=288, filesize=12.0 K 2024-11-12T07:49:36,190 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d56ec85ac1d426368f9bcec18d0d9e10 in 1689ms, sequenceid=288, compaction requested=true 2024-11-12T07:49:36,194 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/recovered.edits/291.seqid, newMaxSeqId=291, maxSeqId=4 2024-11-12T07:49:36,195 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. 2024-11-12T07:49:36,195 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for d56ec85ac1d426368f9bcec18d0d9e10: Waiting for close lock at 1731397774501Running coprocessor pre-close hooks at 1731397774501Disabling compacts and flushes for region at 1731397774501Disabling writes for close at 1731397774501Obtaining lock to block concurrent updates at 1731397774501Preparing flush snapshotting stores in d56ec85ac1d426368f9bcec18d0d9e10 at 1731397774501Finished memstore snapshotting TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731397774502 (+1 ms)Flushing stores of TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10. at 1731397774503 (+1 ms)Flushing d56ec85ac1d426368f9bcec18d0d9e10/A: creating writer at 1731397774503Flushing mob file d56ec85ac1d426368f9bcec18d0d9e10/A: appending metadata at 1731397774510 (+7 ms)Flushing mob file d56ec85ac1d426368f9bcec18d0d9e10/A: closing flushed file at 1731397774510Flushing d56ec85ac1d426368f9bcec18d0d9e10/A: appending metadata at 1731397774929 (+419 ms)Flushing d56ec85ac1d426368f9bcec18d0d9e10/A: closing flushed file at 1731397774929Flushing d56ec85ac1d426368f9bcec18d0d9e10/B: creating writer at 1731397775341 (+412 ms)Flushing d56ec85ac1d426368f9bcec18d0d9e10/B: appending metadata at 1731397775345 (+4 ms)Flushing d56ec85ac1d426368f9bcec18d0d9e10/B: closing flushed file at 1731397775345Flushing d56ec85ac1d426368f9bcec18d0d9e10/C: creating writer at 1731397775759 (+414 ms)Flushing d56ec85ac1d426368f9bcec18d0d9e10/C: appending metadata at 1731397775762 (+3 ms)Flushing d56ec85ac1d426368f9bcec18d0d9e10/C: closing flushed file at 1731397775762Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f94b0f2: reopening flushed file at 1731397776177 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b5d407b: reopening flushed file at 1731397776182 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f6954fd: reopening flushed file at 1731397776186 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d56ec85ac1d426368f9bcec18d0d9e10 in 1689ms, sequenceid=288, compaction requested=true at 1731397776190 (+4 ms)Writing region close event to WAL at 1731397776192 (+2 ms)Running coprocessor post-close hooks at 1731397776195 (+3 ms)Closed at 1731397776195 2024-11-12T07:49:36,197 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,197 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=d56ec85ac1d426368f9bcec18d0d9e10, regionState=CLOSED 2024-11-12T07:49:36,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:49:36,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-12T07:49:36,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure d56ec85ac1d426368f9bcec18d0d9e10, server=f9fc96254ef8,44299,1731397650797 in 1.8520 sec 2024-11-12T07:49:36,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-12T07:49:36,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d56ec85ac1d426368f9bcec18d0d9e10, UNASSIGN in 1.8570 sec 2024-11-12T07:49:36,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-12T07:49:36,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8620 sec 2024-11-12T07:49:36,208 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397776208"}]},"ts":"1731397776208"} 2024-11-12T07:49:36,210 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T07:49:36,210 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T07:49:36,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8750 sec 2024-11-12T07:49:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T07:49:36,489 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:36,491 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T07:49:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:36,496 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T07:49:36,498 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:36,501 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,504 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/recovered.edits] 2024-11-12T07:49:36,507 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/27210f088e004999997b64b6c9aa128b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/27210f088e004999997b64b6c9aa128b 2024-11-12T07:49:36,509 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/4af16289b8fa4ca5a8b633dd26cbbba0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/4af16289b8fa4ca5a8b633dd26cbbba0 2024-11-12T07:49:36,510 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c9fb6d4fca274b2baaff8646ffa9329f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/c9fb6d4fca274b2baaff8646ffa9329f 2024-11-12T07:49:36,512 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/eeb92b9f07b7419bad594ef752162f0a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/A/eeb92b9f07b7419bad594ef752162f0a 2024-11-12T07:49:36,516 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/170af6a6b0ca4114a7bd97c26fe72ea9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/170af6a6b0ca4114a7bd97c26fe72ea9 2024-11-12T07:49:36,518 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/23fd8cb5f9814065a4b67b576681b89b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/23fd8cb5f9814065a4b67b576681b89b 2024-11-12T07:49:36,520 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bb5a3abb258f437a8a68f2ee733b478a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/bb5a3abb258f437a8a68f2ee733b478a 2024-11-12T07:49:36,522 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/c68743ee1a2a43a4b760764752c5614c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/B/c68743ee1a2a43a4b760764752c5614c 2024-11-12T07:49:36,524 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/02e3a46c60654ab9a0295dbf6d426ac5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/02e3a46c60654ab9a0295dbf6d426ac5 2024-11-12T07:49:36,525 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/559c766a7eed42ef8fafca8af4e9c3ff to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/559c766a7eed42ef8fafca8af4e9c3ff 2024-11-12T07:49:36,526 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/aff628673fc24ad0a6cfc96ae7a0b765 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/aff628673fc24ad0a6cfc96ae7a0b765 2024-11-12T07:49:36,527 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/b0cb4ef0983f46e78b98d90a7a8d4d2d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/C/b0cb4ef0983f46e78b98d90a7a8d4d2d 2024-11-12T07:49:36,529 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/recovered.edits/291.seqid to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10/recovered.edits/291.seqid 2024-11-12T07:49:36,530 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,530 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T07:49:36,530 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T07:49:36,531 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T07:49:36,534 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112021a93b3f8064295bad3f422b83a1f27_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112021a93b3f8064295bad3f422b83a1f27_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,535 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111210292a10ad854558b739a890b4f5e3b5_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111210292a10ad854558b739a890b4f5e3b5_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,536 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111216665244a1c647e39e56400ff24dfc60_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111216665244a1c647e39e56400ff24dfc60_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,537 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111221a7b9410311461a83225603a0664b9b_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111221a7b9410311461a83225603a0664b9b_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,538 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112422150d104eb4930b2272a2aad0e66b8_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112422150d104eb4930b2272a2aad0e66b8_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,540 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112593e02dcc1f2448e80a4e973163b06b0_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112593e02dcc1f2448e80a4e973163b06b0_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,541 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112634cf8e7f9a34a178cb3389345dc3868_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112634cf8e7f9a34a178cb3389345dc3868_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,542 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111272da0eda773d4f73bd788adeb2ccaab8_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111272da0eda773d4f73bd788adeb2ccaab8_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,543 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111273d0e31f793d48b7a4be58064232e573_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111273d0e31f793d48b7a4be58064232e573_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,544 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285389de7e0254790a314c436017b5d9c_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285389de7e0254790a314c436017b5d9c_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,545 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293b1b7efe68a4a13a62b680a760bacb4_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111293b1b7efe68a4a13a62b680a760bacb4_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,547 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a8cdd6478e974a71addca0fbcd7eae47_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a8cdd6478e974a71addca0fbcd7eae47_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,548 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b1167853bdfc4a59871b334cc9adc0ed_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b1167853bdfc4a59871b334cc9adc0ed_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,549 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9c6faac9cea4b4dab204d63c05f1947_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9c6faac9cea4b4dab204d63c05f1947_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,550 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dfe71421e4054b2b8dd6f6dc06e1b728_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112dfe71421e4054b2b8dd6f6dc06e1b728_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,551 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee03be3242b24570b9eb78419b7af6e5_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee03be3242b24570b9eb78419b7af6e5_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,553 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee93d1d0a64d4d11b64fc1da27f4e2a6_d56ec85ac1d426368f9bcec18d0d9e10 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee93d1d0a64d4d11b64fc1da27f4e2a6_d56ec85ac1d426368f9bcec18d0d9e10 2024-11-12T07:49:36,553 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T07:49:36,555 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:36,558 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T07:49:36,560 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T07:49:36,562 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:36,562 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T07:49:36,562 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731397776562"}]},"ts":"9223372036854775807"} 2024-11-12T07:49:36,564 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T07:49:36,564 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d56ec85ac1d426368f9bcec18d0d9e10, NAME => 'TestAcidGuarantees,,1731397742711.d56ec85ac1d426368f9bcec18d0d9e10.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T07:49:36,564 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T07:49:36,565 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731397776564"}]},"ts":"9223372036854775807"} 2024-11-12T07:49:36,567 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T07:49:36,568 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:36,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 77 msec 2024-11-12T07:49:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T07:49:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T07:49:36,609 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:36,623 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=338 (was 396), ProcessCount=11 (was 11), AvailableMemoryMB=5822 (was 5924) 2024-11-12T07:49:36,636 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=338, ProcessCount=11, AvailableMemoryMB=5822 2024-11-12T07:49:36,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:49:36,638 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:49:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T07:49:36,640 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:49:36,640 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:36,640 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-12T07:49:36,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-12T07:49:36,642 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:49:36,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742234_1410 (size=1061) 2024-11-12T07:49:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-12T07:49:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-12T07:49:37,053 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f 2024-11-12T07:49:37,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742235_1411 (size=53) 2024-11-12T07:49:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-12T07:49:37,462 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:49:37,462 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e39e5d3040dc37fe65a005881116393f, disabling compactions & flushes 2024-11-12T07:49:37,462 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:37,462 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:37,462 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. after waiting 0 ms 2024-11-12T07:49:37,463 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:37,463 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:37,463 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e39e5d3040dc37fe65a005881116393f: Waiting for close lock at 1731397777462Disabling compacts and flushes for region at 1731397777462Disabling writes for close at 1731397777462Writing region close event to WAL at 1731397777463 (+1 ms)Closed at 1731397777463 2024-11-12T07:49:37,465 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:49:37,466 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731397777465"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397777465"}]},"ts":"1731397777465"} 2024-11-12T07:49:37,469 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:49:37,471 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:49:37,471 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397777471"}]},"ts":"1731397777471"} 2024-11-12T07:49:37,474 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T07:49:37,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, ASSIGN}] 2024-11-12T07:49:37,477 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, ASSIGN 2024-11-12T07:49:37,477 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, ASSIGN; state=OFFLINE, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=false 2024-11-12T07:49:37,628 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=e39e5d3040dc37fe65a005881116393f, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:37,632 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, ASSIGN because future has completed 2024-11-12T07:49:37,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:49:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-12T07:49:37,790 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:37,790 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:49:37,791 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,791 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:49:37,791 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,792 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,793 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,795 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:37,795 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e39e5d3040dc37fe65a005881116393f columnFamilyName A 2024-11-12T07:49:37,795 DEBUG [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:37,796 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.HStore(327): Store=e39e5d3040dc37fe65a005881116393f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:37,796 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,798 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:37,798 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e39e5d3040dc37fe65a005881116393f columnFamilyName B 2024-11-12T07:49:37,798 DEBUG [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:37,799 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.HStore(327): Store=e39e5d3040dc37fe65a005881116393f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:37,799 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,800 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:49:37,801 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e39e5d3040dc37fe65a005881116393f columnFamilyName C 2024-11-12T07:49:37,801 DEBUG [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:49:37,801 INFO [StoreOpener-e39e5d3040dc37fe65a005881116393f-1 {}] regionserver.HStore(327): Store=e39e5d3040dc37fe65a005881116393f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:49:37,802 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:37,802 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,803 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,803 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,805 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,805 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,805 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:49:37,807 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,810 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:49:37,810 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened e39e5d3040dc37fe65a005881116393f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67473318, jitterRate=0.005430787801742554}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:49:37,810 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:37,811 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for e39e5d3040dc37fe65a005881116393f: Running coprocessor pre-open hook at 1731397777792Writing region info on filesystem at 1731397777792Initializing all the Stores at 1731397777793 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397777793Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397777793Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397777793Cleaning up temporary data from old regions at 1731397777805 (+12 ms)Running coprocessor post-open hooks at 1731397777810 (+5 ms)Region opened successfully at 1731397777811 (+1 ms) 2024-11-12T07:49:37,812 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., pid=116, masterSystemTime=1731397777787 2024-11-12T07:49:37,814 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:37,814 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:37,815 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=e39e5d3040dc37fe65a005881116393f, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:37,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:49:37,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-12T07:49:37,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 in 183 msec 2024-11-12T07:49:37,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-12T07:49:37,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, ASSIGN in 343 msec 2024-11-12T07:49:37,820 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:49:37,820 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397777820"}]},"ts":"1731397777820"} 2024-11-12T07:49:37,821 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T07:49:37,822 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:49:37,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-12T07:49:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-12T07:49:38,789 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:38,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,796 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4555df52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,801 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2b4d8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,803 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,805 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:49:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,43319,1731397650023 2024-11-12T07:49:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f91cd88 2024-11-12T07:49:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:49:38,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:49:38,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0bcb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,811 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac48644, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,813 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e245427, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,814 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149df4f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,816 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32567e56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,818 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4708eba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,820 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@418fa17e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3627ce21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,823 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7949b2b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,824 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72427768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,827 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdf07d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f73c183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,831 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5e91d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,833 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,834 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b320f53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e602d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,836 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc19e7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,838 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113ddb2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,839 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8d5974, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:49:38,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:49:38,841 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:49:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:49:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:49:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d861e15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:49:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:49:38,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:38,842 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:49:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c131b52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:49:38,842 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,843 DEBUG [Thread-1874 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,843 DEBUG [Thread-1867 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,843 DEBUG [Thread-1878 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,843 DEBUG [Thread-1876 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,843 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,844 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,844 DEBUG [Thread-1876 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,844 DEBUG [Thread-1865 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,844 DEBUG [Thread-1859 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,845 DEBUG [Thread-1859 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,845 DEBUG [Thread-1876 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,845 DEBUG [Thread-1870 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,845 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-12T07:49:38,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,848 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T07:49:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,849 DEBUG [Thread-1872 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:49:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:49:38,851 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:38,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:49:38,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:49:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2] 2024-11-12T07:49:38,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:38,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:49:38,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:38,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:38,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:38,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:38,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:38,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:38,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/56555bd915524be291972b0bdcff4a8e is 50, key is test_row_0/A:col10/1731397778858/Put/seqid=0 2024-11-12T07:49:38,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742236_1412 (size=14341) 2024-11-12T07:49:38,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731397778874, max timestamp=1731397778884Num uniques 859; 2024-11-12T07:49:38,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T07:49:38,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731397778874, max timestamp=1731397778884Num uniques 859; 2024-11-12T07:49:38,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731397778874, max timestamp=1731397778884Num uniques 859; 2024-11-12T07:49:38,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T07:49:38,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T07:49:38,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397838902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:38,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397838908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397838909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:38,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397838910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:38,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397838911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:38,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T07:49:39,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T07:49:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397839018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397839018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397839027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397839027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397839028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T07:49:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T07:49:39,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397839238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397839238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397839239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397839239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397839239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/56555bd915524be291972b0bdcff4a8e 2024-11-12T07:49:39,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/7dea781080104eeba3a2bfa8d408334d is 50, key is test_row_0/B:col10/1731397778858/Put/seqid=0 2024-11-12T07:49:39,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742237_1413 (size=12001) 2024-11-12T07:49:39,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T07:49:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T07:49:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T07:49:39,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397839548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397839548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397839548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397839548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397839549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:39,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T07:49:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/7dea781080104eeba3a2bfa8d408334d 2024-11-12T07:49:39,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/d0d26fe0b28044bab782433b974555a3 is 50, key is test_row_0/C:col10/1731397778858/Put/seqid=0 2024-11-12T07:49:39,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742238_1414 (size=12001) 2024-11-12T07:49:39,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/d0d26fe0b28044bab782433b974555a3 2024-11-12T07:49:39,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/56555bd915524be291972b0bdcff4a8e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/56555bd915524be291972b0bdcff4a8e 2024-11-12T07:49:39,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/56555bd915524be291972b0bdcff4a8e, entries=200, sequenceid=12, filesize=14.0 K 2024-11-12T07:49:39,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/7dea781080104eeba3a2bfa8d408334d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7dea781080104eeba3a2bfa8d408334d 2024-11-12T07:49:39,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T07:49:39,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:39,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:39,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7dea781080104eeba3a2bfa8d408334d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T07:49:39,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/d0d26fe0b28044bab782433b974555a3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d0d26fe0b28044bab782433b974555a3 2024-11-12T07:49:39,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d0d26fe0b28044bab782433b974555a3, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T07:49:39,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e39e5d3040dc37fe65a005881116393f in 921ms, sequenceid=12, compaction requested=false 2024-11-12T07:49:39,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:39,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T07:49:39,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:39,924 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-12T07:49:39,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:39,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:39,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:39,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:39,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:39,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/9fee473bb4754cf38ae932e4645e0b8f is 50, key is test_row_0/A:col10/1731397778910/Put/seqid=0 2024-11-12T07:49:39,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742239_1415 (size=12001) 2024-11-12T07:49:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T07:49:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:40,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397840068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397840071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397840074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397840075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397840076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397840188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397840188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397840188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397840198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,340 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/9fee473bb4754cf38ae932e4645e0b8f 2024-11-12T07:49:40,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/ffab0233f1414d3686ce3c8e55801d1a is 50, key is test_row_0/B:col10/1731397778910/Put/seqid=0 2024-11-12T07:49:40,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742240_1416 (size=12001) 2024-11-12T07:49:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397840398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397840398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397840398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,405 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397840408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397840707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397840707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397840718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397840718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:40,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:40,762 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/ffab0233f1414d3686ce3c8e55801d1a 2024-11-12T07:49:40,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/cb956afc9371453a95264d73505d1c43 is 50, key is test_row_0/C:col10/1731397778910/Put/seqid=0 2024-11-12T07:49:40,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742241_1417 (size=12001) 2024-11-12T07:49:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T07:49:41,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397841087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,094 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,172 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/cb956afc9371453a95264d73505d1c43 2024-11-12T07:49:41,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/9fee473bb4754cf38ae932e4645e0b8f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9fee473bb4754cf38ae932e4645e0b8f 2024-11-12T07:49:41,179 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9fee473bb4754cf38ae932e4645e0b8f, entries=150, sequenceid=45, filesize=11.7 K 2024-11-12T07:49:41,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/ffab0233f1414d3686ce3c8e55801d1a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/ffab0233f1414d3686ce3c8e55801d1a 2024-11-12T07:49:41,183 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/ffab0233f1414d3686ce3c8e55801d1a, entries=150, sequenceid=45, filesize=11.7 K 2024-11-12T07:49:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/cb956afc9371453a95264d73505d1c43 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/cb956afc9371453a95264d73505d1c43 2024-11-12T07:49:41,187 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/cb956afc9371453a95264d73505d1c43, entries=150, sequenceid=45, filesize=11.7 K 2024-11-12T07:49:41,188 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for e39e5d3040dc37fe65a005881116393f in 1265ms, sequenceid=45, compaction requested=false 2024-11-12T07:49:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-12T07:49:41,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-12T07:49:41,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-12T07:49:41,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-12T07:49:41,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.3460 sec 2024-11-12T07:49:41,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:41,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:49:41,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:41,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:41,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:41,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:41,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:41,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:41,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/5c9d49ed4a3b420db15b6e9dcfbc66e9 is 50, key is test_row_0/A:col10/1731397781231/Put/seqid=0 2024-11-12T07:49:41,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742242_1418 (size=14341) 2024-11-12T07:49:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397841261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397841262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397841268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397841269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397841388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397841388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397841388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397841388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397841598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397841598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397841598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397841598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/5c9d49ed4a3b420db15b6e9dcfbc66e9 2024-11-12T07:49:41,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/7e9daaa9578e49c9a6aaa90173b9246a is 50, key is test_row_0/B:col10/1731397781231/Put/seqid=0 2024-11-12T07:49:41,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742243_1419 (size=12001) 2024-11-12T07:49:41,712 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:49:41,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397841908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397841908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397841908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397841908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/7e9daaa9578e49c9a6aaa90173b9246a 2024-11-12T07:49:42,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/08bb155eb8fb44d3abbf42524b535f34 is 50, key is test_row_0/C:col10/1731397781231/Put/seqid=0 2024-11-12T07:49:42,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742244_1420 (size=12001) 2024-11-12T07:49:42,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/08bb155eb8fb44d3abbf42524b535f34 2024-11-12T07:49:42,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/5c9d49ed4a3b420db15b6e9dcfbc66e9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5c9d49ed4a3b420db15b6e9dcfbc66e9 2024-11-12T07:49:42,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5c9d49ed4a3b420db15b6e9dcfbc66e9, entries=200, sequenceid=57, filesize=14.0 K 2024-11-12T07:49:42,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/7e9daaa9578e49c9a6aaa90173b9246a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7e9daaa9578e49c9a6aaa90173b9246a 2024-11-12T07:49:42,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7e9daaa9578e49c9a6aaa90173b9246a, entries=150, sequenceid=57, filesize=11.7 K 2024-11-12T07:49:42,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/08bb155eb8fb44d3abbf42524b535f34 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/08bb155eb8fb44d3abbf42524b535f34 2024-11-12T07:49:42,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/08bb155eb8fb44d3abbf42524b535f34, entries=150, sequenceid=57, filesize=11.7 K 2024-11-12T07:49:42,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e39e5d3040dc37fe65a005881116393f in 845ms, sequenceid=57, compaction requested=true 2024-11-12T07:49:42,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:42,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:42,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:42,079 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:42,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:42,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:42,079 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:42,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:42,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:42,079 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:42,080 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:42,080 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/A is initiating minor compaction (all files) 2024-11-12T07:49:42,080 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/B is initiating minor compaction (all files) 2024-11-12T07:49:42,080 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/A in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:42,080 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/B in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:42,080 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/56555bd915524be291972b0bdcff4a8e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9fee473bb4754cf38ae932e4645e0b8f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5c9d49ed4a3b420db15b6e9dcfbc66e9] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=39.7 K 2024-11-12T07:49:42,080 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7dea781080104eeba3a2bfa8d408334d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/ffab0233f1414d3686ce3c8e55801d1a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7e9daaa9578e49c9a6aaa90173b9246a] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=35.2 K 2024-11-12T07:49:42,080 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56555bd915524be291972b0bdcff4a8e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731397778853 2024-11-12T07:49:42,080 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dea781080104eeba3a2bfa8d408334d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731397778853 2024-11-12T07:49:42,080 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ffab0233f1414d3686ce3c8e55801d1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731397778907 2024-11-12T07:49:42,080 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fee473bb4754cf38ae932e4645e0b8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731397778907 2024-11-12T07:49:42,081 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9daaa9578e49c9a6aaa90173b9246a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731397780061 2024-11-12T07:49:42,081 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c9d49ed4a3b420db15b6e9dcfbc66e9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731397780061 2024-11-12T07:49:42,092 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#A#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:42,092 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#B#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:42,093 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/715a801b14f84d49bf9a4df8f7931678 is 50, key is test_row_0/B:col10/1731397781231/Put/seqid=0 2024-11-12T07:49:42,093 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/da2eaf9278f341d6b6894e5bf3dcc944 is 50, key is test_row_0/A:col10/1731397781231/Put/seqid=0 2024-11-12T07:49:42,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742245_1421 (size=12104) 2024-11-12T07:49:42,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742246_1422 (size=12104) 2024-11-12T07:49:42,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:42,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T07:49:42,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:42,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:42,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:42,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:42,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:42,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:42,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/435bb31bc22f4ba9aef20271927db115 is 50, key is test_row_0/A:col10/1731397782430/Put/seqid=0 2024-11-12T07:49:42,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742247_1423 (size=14341) 2024-11-12T07:49:42,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397842446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397842467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397842467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397842467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,501 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/da2eaf9278f341d6b6894e5bf3dcc944 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/da2eaf9278f341d6b6894e5bf3dcc944 2024-11-12T07:49:42,505 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/A of e39e5d3040dc37fe65a005881116393f into da2eaf9278f341d6b6894e5bf3dcc944(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:42,505 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:42,505 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/A, priority=13, startTime=1731397782078; duration=0sec 2024-11-12T07:49:42,505 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:42,505 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:A 2024-11-12T07:49:42,505 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:42,506 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:42,506 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/C is initiating minor compaction (all files) 2024-11-12T07:49:42,506 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/C in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:42,506 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d0d26fe0b28044bab782433b974555a3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/cb956afc9371453a95264d73505d1c43, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/08bb155eb8fb44d3abbf42524b535f34] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=35.2 K 2024-11-12T07:49:42,507 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d26fe0b28044bab782433b974555a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731397778853 2024-11-12T07:49:42,507 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb956afc9371453a95264d73505d1c43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731397778907 2024-11-12T07:49:42,507 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08bb155eb8fb44d3abbf42524b535f34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731397780061 2024-11-12T07:49:42,509 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/715a801b14f84d49bf9a4df8f7931678 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/715a801b14f84d49bf9a4df8f7931678 2024-11-12T07:49:42,512 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/B of e39e5d3040dc37fe65a005881116393f into 715a801b14f84d49bf9a4df8f7931678(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:42,512 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:42,512 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/B, priority=13, startTime=1731397782079; duration=0sec 2024-11-12T07:49:42,512 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#C#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:42,512 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:42,512 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:B 2024-11-12T07:49:42,512 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/6874181875104cffa5a438026a2b88f7 is 50, key is test_row_0/C:col10/1731397781231/Put/seqid=0 2024-11-12T07:49:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742248_1424 (size=12104) 2024-11-12T07:49:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397842577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397842587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397842588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397842588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397842787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397842797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397842798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397842798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:42,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/435bb31bc22f4ba9aef20271927db115 2024-11-12T07:49:42,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/a5e0cdfde15a4d94bce1901ad7b6f14b is 50, key is test_row_0/B:col10/1731397782430/Put/seqid=0 2024-11-12T07:49:42,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742249_1425 (size=12001) 2024-11-12T07:49:42,918 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/6874181875104cffa5a438026a2b88f7 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6874181875104cffa5a438026a2b88f7 2024-11-12T07:49:42,922 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/C of e39e5d3040dc37fe65a005881116393f into 6874181875104cffa5a438026a2b88f7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:42,922 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:42,922 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/C, priority=13, startTime=1731397782079; duration=0sec 2024-11-12T07:49:42,922 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:42,922 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:C 2024-11-12T07:49:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T07:49:43,008 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:43,008 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-12T07:49:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T07:49:43,009 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:43,010 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:43,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397843098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397843107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397843107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397843107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,112 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397843107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T07:49:43,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T07:49:43,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:43,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/a5e0cdfde15a4d94bce1901ad7b6f14b 2024-11-12T07:49:43,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/86a1919d9e0249edbd787175c2efc354 is 50, key is test_row_0/C:col10/1731397782430/Put/seqid=0 2024-11-12T07:49:43,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742250_1426 (size=12001) 2024-11-12T07:49:43,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T07:49:43,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:43,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T07:49:43,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T07:49:43,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:43,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397843607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T07:49:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:43,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397843618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397843628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397843628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T07:49:43,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/86a1919d9e0249edbd787175c2efc354 2024-11-12T07:49:43,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/435bb31bc22f4ba9aef20271927db115 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/435bb31bc22f4ba9aef20271927db115 2024-11-12T07:49:43,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/435bb31bc22f4ba9aef20271927db115, entries=200, sequenceid=78, filesize=14.0 K 2024-11-12T07:49:43,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/a5e0cdfde15a4d94bce1901ad7b6f14b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5e0cdfde15a4d94bce1901ad7b6f14b 2024-11-12T07:49:43,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5e0cdfde15a4d94bce1901ad7b6f14b, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T07:49:43,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/86a1919d9e0249edbd787175c2efc354 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/86a1919d9e0249edbd787175c2efc354 2024-11-12T07:49:43,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/86a1919d9e0249edbd787175c2efc354, entries=150, sequenceid=78, filesize=11.7 K 2024-11-12T07:49:43,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for e39e5d3040dc37fe65a005881116393f in 1243ms, sequenceid=78, compaction requested=false 2024-11-12T07:49:43,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:43,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T07:49:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:43,773 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:49:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:43,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/c861983b1ae649eb990aa9938d931806 is 50, key is test_row_0/A:col10/1731397782446/Put/seqid=0 2024-11-12T07:49:43,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742251_1427 (size=12001) 2024-11-12T07:49:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T07:49:44,180 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/c861983b1ae649eb990aa9938d931806 2024-11-12T07:49:44,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/813a71c4d0474130bbb493c9bbda14e1 is 50, key is test_row_0/B:col10/1731397782446/Put/seqid=0 2024-11-12T07:49:44,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742252_1428 (size=12001) 2024-11-12T07:49:44,595 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/813a71c4d0474130bbb493c9bbda14e1 2024-11-12T07:49:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/99e64e3024bd4460be504330f146d6ab is 50, key is test_row_0/C:col10/1731397782446/Put/seqid=0 2024-11-12T07:49:44,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742253_1429 (size=12001) 2024-11-12T07:49:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:44,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:44,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397844667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:44,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731397784654, max timestamp=1731397784676Num uniques 859; 2024-11-12T07:49:44,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T07:49:44,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731397784654, max timestamp=1731397784676Num uniques 859; 2024-11-12T07:49:44,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731397784654, max timestamp=1731397784676Num uniques 859; 2024-11-12T07:49:44,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T07:49:44,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T07:49:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:44,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397844703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397844704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:44,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397844705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:44,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397844788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:44,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397844817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:44,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397844817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397844828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,005 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/99e64e3024bd4460be504330f146d6ab 2024-11-12T07:49:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/c861983b1ae649eb990aa9938d931806 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/c861983b1ae649eb990aa9938d931806 2024-11-12T07:49:45,012 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/c861983b1ae649eb990aa9938d931806, entries=150, sequenceid=93, filesize=11.7 K 2024-11-12T07:49:45,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/813a71c4d0474130bbb493c9bbda14e1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/813a71c4d0474130bbb493c9bbda14e1 2024-11-12T07:49:45,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397845008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,016 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/813a71c4d0474130bbb493c9bbda14e1, entries=150, sequenceid=93, filesize=11.7 K 2024-11-12T07:49:45,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/99e64e3024bd4460be504330f146d6ab as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/99e64e3024bd4460be504330f146d6ab 2024-11-12T07:49:45,019 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/99e64e3024bd4460be504330f146d6ab, entries=150, sequenceid=93, filesize=11.7 K 2024-11-12T07:49:45,020 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for e39e5d3040dc37fe65a005881116393f in 1247ms, sequenceid=93, compaction requested=true 2024-11-12T07:49:45,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:45,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-12T07:49:45,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-12T07:49:45,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-12T07:49:45,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-11-12T07:49:45,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.0150 sec 2024-11-12T07:49:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:45,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-12T07:49:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:45,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:45,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/7f9a0149a4ab471eae1aff9fedfeac79 is 50, key is test_row_0/A:col10/1731397784704/Put/seqid=0 2024-11-12T07:49:45,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742254_1430 (size=14341) 2024-11-12T07:49:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397845059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397845065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397845066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T07:49:45,168 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:45,169 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-12T07:49:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T07:49:45,170 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:45,171 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:45,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:45,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397845177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397845177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397845188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T07:49:45,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T07:49:45,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:45,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397845328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397845387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397845388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397845408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/7f9a0149a4ab471eae1aff9fedfeac79 2024-11-12T07:49:45,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/5d1282b576ef4041a649887cfbb0c7a5 is 50, key is test_row_0/B:col10/1731397784704/Put/seqid=0 2024-11-12T07:49:45,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742255_1431 (size=12001) 2024-11-12T07:49:45,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T07:49:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T07:49:45,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T07:49:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397845698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397845698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397845728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T07:49:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T07:49:45,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397845848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:45,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:45,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/5d1282b576ef4041a649887cfbb0c7a5 2024-11-12T07:49:45,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/2014e53fdc38453ab770013b4933461d is 50, key is test_row_0/C:col10/1731397784704/Put/seqid=0 2024-11-12T07:49:45,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742256_1432 (size=12001) 2024-11-12T07:49:45,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/2014e53fdc38453ab770013b4933461d 2024-11-12T07:49:45,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/7f9a0149a4ab471eae1aff9fedfeac79 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/7f9a0149a4ab471eae1aff9fedfeac79 2024-11-12T07:49:45,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/7f9a0149a4ab471eae1aff9fedfeac79, entries=200, sequenceid=126, filesize=14.0 K 2024-11-12T07:49:45,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/5d1282b576ef4041a649887cfbb0c7a5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5d1282b576ef4041a649887cfbb0c7a5 2024-11-12T07:49:45,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5d1282b576ef4041a649887cfbb0c7a5, entries=150, sequenceid=126, filesize=11.7 K 2024-11-12T07:49:45,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/2014e53fdc38453ab770013b4933461d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2014e53fdc38453ab770013b4933461d 2024-11-12T07:49:45,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2014e53fdc38453ab770013b4933461d, entries=150, sequenceid=126, filesize=11.7 K 2024-11-12T07:49:45,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for e39e5d3040dc37fe65a005881116393f in 853ms, sequenceid=126, compaction requested=true 2024-11-12T07:49:45,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:45,889 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:49:45,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:45,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:45,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:45,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:45,890 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:49:45,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:45,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:45,890 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:49:45,890 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/A is initiating minor compaction (all files) 2024-11-12T07:49:45,890 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/A in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,891 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:49:45,891 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/da2eaf9278f341d6b6894e5bf3dcc944, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/435bb31bc22f4ba9aef20271927db115, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/c861983b1ae649eb990aa9938d931806, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/7f9a0149a4ab471eae1aff9fedfeac79] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=51.5 K 2024-11-12T07:49:45,891 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/B is initiating minor compaction (all files) 2024-11-12T07:49:45,891 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/B in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,891 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/715a801b14f84d49bf9a4df8f7931678, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5e0cdfde15a4d94bce1901ad7b6f14b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/813a71c4d0474130bbb493c9bbda14e1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5d1282b576ef4041a649887cfbb0c7a5] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=47.0 K 2024-11-12T07:49:45,891 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting da2eaf9278f341d6b6894e5bf3dcc944, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731397780061 2024-11-12T07:49:45,891 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 715a801b14f84d49bf9a4df8f7931678, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731397780061 2024-11-12T07:49:45,891 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 435bb31bc22f4ba9aef20271927db115, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731397781260 2024-11-12T07:49:45,891 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting c861983b1ae649eb990aa9938d931806, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731397782446 2024-11-12T07:49:45,891 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e0cdfde15a4d94bce1901ad7b6f14b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731397781260 2024-11-12T07:49:45,892 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f9a0149a4ab471eae1aff9fedfeac79, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731397784698 2024-11-12T07:49:45,892 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 813a71c4d0474130bbb493c9bbda14e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731397782446 2024-11-12T07:49:45,892 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1282b576ef4041a649887cfbb0c7a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731397784702 2024-11-12T07:49:45,913 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#A#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:45,913 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/21dffdb35e364f22b97a2c07f2b40a22 is 50, key is test_row_0/A:col10/1731397784704/Put/seqid=0 2024-11-12T07:49:45,922 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#B#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:45,923 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/a5cab41903354d649b21adb92d2228dd is 50, key is test_row_0/B:col10/1731397784704/Put/seqid=0 2024-11-12T07:49:45,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742258_1434 (size=12241) 2024-11-12T07:49:45,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742257_1433 (size=12241) 2024-11-12T07:49:45,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T07:49:45,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,934 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T07:49:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:45,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:45,938 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/21dffdb35e364f22b97a2c07f2b40a22 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/21dffdb35e364f22b97a2c07f2b40a22 2024-11-12T07:49:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/9ea854a72472445f964a7b84fce6e668 is 50, key is test_row_0/A:col10/1731397785047/Put/seqid=0 2024-11-12T07:49:45,948 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e39e5d3040dc37fe65a005881116393f/A of e39e5d3040dc37fe65a005881116393f into 21dffdb35e364f22b97a2c07f2b40a22(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:45,948 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:45,948 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/A, priority=12, startTime=1731397785889; duration=0sec 2024-11-12T07:49:45,948 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:45,948 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:A 2024-11-12T07:49:45,948 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:49:45,949 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:49:45,949 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/C is initiating minor compaction (all files) 2024-11-12T07:49:45,950 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/C in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:45,950 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6874181875104cffa5a438026a2b88f7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/86a1919d9e0249edbd787175c2efc354, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/99e64e3024bd4460be504330f146d6ab, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2014e53fdc38453ab770013b4933461d] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=47.0 K 2024-11-12T07:49:45,950 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6874181875104cffa5a438026a2b88f7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731397780061 2024-11-12T07:49:45,951 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a1919d9e0249edbd787175c2efc354, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731397781260 2024-11-12T07:49:45,951 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99e64e3024bd4460be504330f146d6ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731397782446 2024-11-12T07:49:45,951 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2014e53fdc38453ab770013b4933461d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731397784702 2024-11-12T07:49:45,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742259_1435 (size=12151) 2024-11-12T07:49:45,956 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/9ea854a72472445f964a7b84fce6e668 2024-11-12T07:49:45,965 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:45,965 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/ce8342ee5abd4486b425666cd9f317c3 is 50, key is test_row_0/C:col10/1731397784704/Put/seqid=0 2024-11-12T07:49:45,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/1b24a1e693f24709b7b28294cf2a0a6c is 50, key is test_row_0/B:col10/1731397785047/Put/seqid=0 2024-11-12T07:49:45,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742260_1436 (size=12241) 2024-11-12T07:49:45,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742261_1437 (size=12151) 2024-11-12T07:49:45,985 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/ce8342ee5abd4486b425666cd9f317c3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ce8342ee5abd4486b425666cd9f317c3 2024-11-12T07:49:45,989 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e39e5d3040dc37fe65a005881116393f/C of e39e5d3040dc37fe65a005881116393f into ce8342ee5abd4486b425666cd9f317c3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:45,989 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:45,989 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/C, priority=12, startTime=1731397785890; duration=0sec 2024-11-12T07:49:45,989 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:45,989 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:C 2024-11-12T07:49:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:46,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397846258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397786244, max timestamp=1731397786264Num uniques 859; 2024-11-12T07:49:46,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397786244, max timestamp=1731397786264Num uniques 859; 2024-11-12T07:49:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:46,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397786244, max timestamp=1731397786264Num uniques 859; 2024-11-12T07:49:46,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T07:49:46,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397846324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,335 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/a5cab41903354d649b21adb92d2228dd as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5cab41903354d649b21adb92d2228dd 2024-11-12T07:49:46,338 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e39e5d3040dc37fe65a005881116393f/B of e39e5d3040dc37fe65a005881116393f into a5cab41903354d649b21adb92d2228dd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:46,339 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:46,339 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/B, priority=12, startTime=1731397785890; duration=0sec 2024-11-12T07:49:46,339 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,339 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:B 2024-11-12T07:49:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397846328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,380 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/1b24a1e693f24709b7b28294cf2a0a6c 2024-11-12T07:49:46,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/5f1d7e98b23d4ad3877ef48079ae4de0 is 50, key is test_row_0/C:col10/1731397785047/Put/seqid=0 2024-11-12T07:49:46,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397846378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742262_1438 (size=12151) 2024-11-12T07:49:46,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397846438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397846447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397846597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397846648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397846658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,790 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/5f1d7e98b23d4ad3877ef48079ae4de0 2024-11-12T07:49:46,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/9ea854a72472445f964a7b84fce6e668 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9ea854a72472445f964a7b84fce6e668 2024-11-12T07:49:46,796 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9ea854a72472445f964a7b84fce6e668, entries=150, sequenceid=137, filesize=11.9 K 2024-11-12T07:49:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/1b24a1e693f24709b7b28294cf2a0a6c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/1b24a1e693f24709b7b28294cf2a0a6c 2024-11-12T07:49:46,800 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/1b24a1e693f24709b7b28294cf2a0a6c, entries=150, sequenceid=137, filesize=11.9 K 2024-11-12T07:49:46,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/5f1d7e98b23d4ad3877ef48079ae4de0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5f1d7e98b23d4ad3877ef48079ae4de0 2024-11-12T07:49:46,804 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5f1d7e98b23d4ad3877ef48079ae4de0, entries=150, sequenceid=137, filesize=11.9 K 2024-11-12T07:49:46,805 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e39e5d3040dc37fe65a005881116393f in 870ms, sequenceid=137, compaction requested=false 2024-11-12T07:49:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:46,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-12T07:49:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-12T07:49:46,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-12T07:49:46,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6360 sec 2024-11-12T07:49:46,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6410 sec 2024-11-12T07:49:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:46,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T07:49:46,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:46,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:46,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:46,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:46,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:46,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:46,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/69a88b6815d44073bde3cb3612cecc79 is 50, key is test_row_0/A:col10/1731397786314/Put/seqid=0 2024-11-12T07:49:46,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742263_1439 (size=14541) 2024-11-12T07:49:46,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397846918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397846919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397846957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:46,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397846968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397847028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397847037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397847157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,165 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397847238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397847247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/69a88b6815d44073bde3cb3612cecc79 2024-11-12T07:49:47,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/56339212ff8544738ee3dacb76aa58b0 is 50, key is test_row_0/B:col10/1731397786314/Put/seqid=0 2024-11-12T07:49:47,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742264_1440 (size=12151) 2024-11-12T07:49:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T07:49:47,329 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:47,329 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-12T07:49:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T07:49:47,331 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:47,331 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:47,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T07:49:47,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T07:49:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:47,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397847477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397847488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397847557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397847557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:47,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T07:49:47,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:47,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:47,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:47,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:47,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T07:49:47,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/56339212ff8544738ee3dacb76aa58b0 2024-11-12T07:49:47,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/643a09363c954387a223c4820a4f063d is 50, key is test_row_0/C:col10/1731397786314/Put/seqid=0 2024-11-12T07:49:47,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742265_1441 (size=12151) 2024-11-12T07:49:47,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/643a09363c954387a223c4820a4f063d 2024-11-12T07:49:47,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/69a88b6815d44073bde3cb3612cecc79 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/69a88b6815d44073bde3cb3612cecc79 2024-11-12T07:49:47,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/69a88b6815d44073bde3cb3612cecc79, entries=200, sequenceid=177, filesize=14.2 K 2024-11-12T07:49:47,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/56339212ff8544738ee3dacb76aa58b0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/56339212ff8544738ee3dacb76aa58b0 2024-11-12T07:49:47,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/56339212ff8544738ee3dacb76aa58b0, entries=150, sequenceid=177, filesize=11.9 K 2024-11-12T07:49:47,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/643a09363c954387a223c4820a4f063d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/643a09363c954387a223c4820a4f063d 2024-11-12T07:49:47,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/643a09363c954387a223c4820a4f063d, entries=150, sequenceid=177, filesize=11.9 K 2024-11-12T07:49:47,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for e39e5d3040dc37fe65a005881116393f in 853ms, sequenceid=177, compaction requested=true 2024-11-12T07:49:47,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:47,728 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:47,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:47,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:47,728 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:47,728 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/A is initiating minor compaction (all files) 2024-11-12T07:49:47,729 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/A in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:47,729 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/21dffdb35e364f22b97a2c07f2b40a22, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9ea854a72472445f964a7b84fce6e668, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/69a88b6815d44073bde3cb3612cecc79] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=38.0 K 2024-11-12T07:49:47,729 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:47,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:47,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:47,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:47,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:47,729 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21dffdb35e364f22b97a2c07f2b40a22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731397784702 2024-11-12T07:49:47,730 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:47,730 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea854a72472445f964a7b84fce6e668, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731397785047 2024-11-12T07:49:47,730 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/B is initiating minor compaction (all files) 2024-11-12T07:49:47,730 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/B in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:47,730 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69a88b6815d44073bde3cb3612cecc79, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397786314 2024-11-12T07:49:47,730 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5cab41903354d649b21adb92d2228dd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/1b24a1e693f24709b7b28294cf2a0a6c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/56339212ff8544738ee3dacb76aa58b0] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=35.7 K 2024-11-12T07:49:47,730 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting a5cab41903354d649b21adb92d2228dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731397784702 2024-11-12T07:49:47,731 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b24a1e693f24709b7b28294cf2a0a6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731397785047 2024-11-12T07:49:47,731 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 56339212ff8544738ee3dacb76aa58b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397786314 2024-11-12T07:49:47,737 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#A#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:47,738 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/f8e7be3452f14fc8b9d5c6bea94789ab is 50, key is test_row_0/A:col10/1731397786314/Put/seqid=0 2024-11-12T07:49:47,740 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#B#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:47,741 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/dfe3ffb44e7a4882be95a4929162574b is 50, key is test_row_0/B:col10/1731397786314/Put/seqid=0 2024-11-12T07:49:47,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742267_1443 (size=12493) 2024-11-12T07:49:47,753 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/dfe3ffb44e7a4882be95a4929162574b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/dfe3ffb44e7a4882be95a4929162574b 2024-11-12T07:49:47,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742266_1442 (size=12493) 2024-11-12T07:49:47,762 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/B of e39e5d3040dc37fe65a005881116393f into dfe3ffb44e7a4882be95a4929162574b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:47,762 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:47,762 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/B, priority=13, startTime=1731397787729; duration=0sec 2024-11-12T07:49:47,762 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:47,763 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:B 2024-11-12T07:49:47,763 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:47,764 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:47,764 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/C is initiating minor compaction (all files) 2024-11-12T07:49:47,764 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/C in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:47,764 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ce8342ee5abd4486b425666cd9f317c3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5f1d7e98b23d4ad3877ef48079ae4de0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/643a09363c954387a223c4820a4f063d] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=35.7 K 2024-11-12T07:49:47,764 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ce8342ee5abd4486b425666cd9f317c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731397784702 2024-11-12T07:49:47,765 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f1d7e98b23d4ad3877ef48079ae4de0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731397785047 2024-11-12T07:49:47,765 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 643a09363c954387a223c4820a4f063d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397786314 2024-11-12T07:49:47,773 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#C#compaction#374 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:47,773 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/21a32f061baf4081947893425d93db66 is 50, key is test_row_0/C:col10/1731397786314/Put/seqid=0 2024-11-12T07:49:47,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742268_1444 (size=12493) 2024-11-12T07:49:47,786 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/21a32f061baf4081947893425d93db66 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/21a32f061baf4081947893425d93db66 2024-11-12T07:49:47,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T07:49:47,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:47,789 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:49:47,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:47,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:47,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:47,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:47,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:47,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:47,795 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/C of e39e5d3040dc37fe65a005881116393f into 21a32f061baf4081947893425d93db66(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:47,795 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:47,795 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/C, priority=13, startTime=1731397787729; duration=0sec 2024-11-12T07:49:47,795 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:47,795 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:C 2024-11-12T07:49:47,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/eb0b5d1c8790472ab776135f97d9900f is 50, key is test_row_0/A:col10/1731397786917/Put/seqid=0 2024-11-12T07:49:47,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742269_1445 (size=12151) 2024-11-12T07:49:47,807 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/eb0b5d1c8790472ab776135f97d9900f 2024-11-12T07:49:47,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/95a42063732c48f3ac0543216e72bfbd is 50, key is test_row_0/B:col10/1731397786917/Put/seqid=0 2024-11-12T07:49:47,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742270_1446 (size=12151) 2024-11-12T07:49:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T07:49:48,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:48,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397788108, max timestamp=1731397788157Num uniques 859; 2024-11-12T07:49:48,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397788108, max timestamp=1731397788157Num uniques 859; 2024-11-12T07:49:48,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:48,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:48,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397788108, max timestamp=1731397788157Num uniques 859; 2024-11-12T07:49:48,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:48,164 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/f8e7be3452f14fc8b9d5c6bea94789ab as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f8e7be3452f14fc8b9d5c6bea94789ab 2024-11-12T07:49:48,167 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/A of e39e5d3040dc37fe65a005881116393f into f8e7be3452f14fc8b9d5c6bea94789ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:48,167 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:48,167 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/A, priority=13, startTime=1731397787727; duration=0sec 2024-11-12T07:49:48,167 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:48,167 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:A 2024-11-12T07:49:48,226 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/95a42063732c48f3ac0543216e72bfbd 2024-11-12T07:49:48,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/6241f0ff39574455987546cebd6b7303 is 50, key is test_row_0/C:col10/1731397786917/Put/seqid=0 2024-11-12T07:49:48,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742271_1447 (size=12151) 2024-11-12T07:49:48,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397848220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397848224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397848348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397848348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T07:49:48,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397848498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,506 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397848507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,513 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_0' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397848568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397848568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,636 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/6241f0ff39574455987546cebd6b7303 2024-11-12T07:49:48,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/eb0b5d1c8790472ab776135f97d9900f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eb0b5d1c8790472ab776135f97d9900f 2024-11-12T07:49:48,642 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eb0b5d1c8790472ab776135f97d9900f, entries=150, sequenceid=188, filesize=11.9 K 2024-11-12T07:49:48,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/95a42063732c48f3ac0543216e72bfbd as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/95a42063732c48f3ac0543216e72bfbd 2024-11-12T07:49:48,645 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/95a42063732c48f3ac0543216e72bfbd, entries=150, sequenceid=188, filesize=11.9 K 2024-11-12T07:49:48,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/6241f0ff39574455987546cebd6b7303 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6241f0ff39574455987546cebd6b7303 2024-11-12T07:49:48,648 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6241f0ff39574455987546cebd6b7303, entries=150, sequenceid=188, filesize=11.9 K 2024-11-12T07:49:48,648 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e39e5d3040dc37fe65a005881116393f in 859ms, sequenceid=188, compaction requested=false 2024-11-12T07:49:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-12T07:49:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-12T07:49:48,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-12T07:49:48,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3190 sec 2024-11-12T07:49:48,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3230 sec 2024-11-12T07:49:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:48,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T07:49:48,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:48,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:48,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:48,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:48,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:48,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:48,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/215b2baccce141f78f6e60495dc55222 is 50, key is test_row_0/A:col10/1731397788217/Put/seqid=0 2024-11-12T07:49:48,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742272_1448 (size=14541) 2024-11-12T07:49:48,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397848901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:48,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397848906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:48,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397849018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397849018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397849228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397849228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:49,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/215b2baccce141f78f6e60495dc55222 2024-11-12T07:49:49,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/5a5aec526c3043bdb9805cfc415c50e8 is 50, key is test_row_0/B:col10/1731397788217/Put/seqid=0 2024-11-12T07:49:49,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742273_1449 (size=12151) 2024-11-12T07:49:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T07:49:49,478 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:49,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-12T07:49:49,480 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T07:49:49,481 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:49,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:49,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397849538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:49,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397849538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:49,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T07:49:49,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T07:49:49,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:49,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:49,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:49,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:49,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:49,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/5a5aec526c3043bdb9805cfc415c50e8 2024-11-12T07:49:49,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/2793cdcb05564658b799db51771bc43a is 50, key is test_row_0/C:col10/1731397788217/Put/seqid=0 2024-11-12T07:49:49,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742274_1450 (size=12151) 2024-11-12T07:49:49,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T07:49:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:49,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:49,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T07:49:49,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T07:49:49,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:49,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:49,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:49,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:49,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:50,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397850048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397850057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:50,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T07:49:50,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:50,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:50,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:50,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:50,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:50,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T07:49:50,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/2793cdcb05564658b799db51771bc43a 2024-11-12T07:49:50,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/215b2baccce141f78f6e60495dc55222 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/215b2baccce141f78f6e60495dc55222 2024-11-12T07:49:50,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/215b2baccce141f78f6e60495dc55222, entries=200, sequenceid=228, filesize=14.2 K 2024-11-12T07:49:50,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/5a5aec526c3043bdb9805cfc415c50e8 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5a5aec526c3043bdb9805cfc415c50e8 2024-11-12T07:49:50,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5a5aec526c3043bdb9805cfc415c50e8, entries=150, sequenceid=228, filesize=11.9 K 2024-11-12T07:49:50,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/2793cdcb05564658b799db51771bc43a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2793cdcb05564658b799db51771bc43a 2024-11-12T07:49:50,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2793cdcb05564658b799db51771bc43a, entries=150, sequenceid=228, filesize=11.9 K 2024-11-12T07:49:50,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e39e5d3040dc37fe65a005881116393f in 1245ms, sequenceid=228, compaction requested=true 2024-11-12T07:49:50,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:50,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:50,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:50,129 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:50,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:50,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:50,129 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:50,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:50,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/A is initiating minor compaction (all files) 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/B is initiating minor compaction (all files) 2024-11-12T07:49:50,130 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/B in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:50,130 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/A in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:50,130 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/dfe3ffb44e7a4882be95a4929162574b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/95a42063732c48f3ac0543216e72bfbd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5a5aec526c3043bdb9805cfc415c50e8] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=35.9 K 2024-11-12T07:49:50,130 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f8e7be3452f14fc8b9d5c6bea94789ab, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eb0b5d1c8790472ab776135f97d9900f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/215b2baccce141f78f6e60495dc55222] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=38.3 K 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe3ffb44e7a4882be95a4929162574b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397786314 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8e7be3452f14fc8b9d5c6bea94789ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397786314 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb0b5d1c8790472ab776135f97d9900f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731397786880 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 95a42063732c48f3ac0543216e72bfbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731397786880 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 215b2baccce141f78f6e60495dc55222, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731397788217 2024-11-12T07:49:50,130 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a5aec526c3043bdb9805cfc415c50e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731397788217 2024-11-12T07:49:50,142 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:50,143 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/a4d676454926429ba808938e6765c92b is 50, key is test_row_0/B:col10/1731397788217/Put/seqid=0 2024-11-12T07:49:50,144 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#A#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:50,144 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/48a82fbad87f4acea44332ffa1f9f6c3 is 50, key is test_row_0/A:col10/1731397788217/Put/seqid=0 2024-11-12T07:49:50,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742275_1451 (size=12595) 2024-11-12T07:49:50,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742276_1452 (size=12595) 2024-11-12T07:49:50,154 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/48a82fbad87f4acea44332ffa1f9f6c3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/48a82fbad87f4acea44332ffa1f9f6c3 2024-11-12T07:49:50,158 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/A of e39e5d3040dc37fe65a005881116393f into 48a82fbad87f4acea44332ffa1f9f6c3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:50,158 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:50,158 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/A, priority=13, startTime=1731397790129; duration=0sec 2024-11-12T07:49:50,158 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:50,158 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:A 2024-11-12T07:49:50,158 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:50,159 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:50,159 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/C is initiating minor compaction (all files) 2024-11-12T07:49:50,159 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/C in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:50,159 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/21a32f061baf4081947893425d93db66, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6241f0ff39574455987546cebd6b7303, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2793cdcb05564658b799db51771bc43a] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=35.9 K 2024-11-12T07:49:50,171 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a32f061baf4081947893425d93db66, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731397786314 2024-11-12T07:49:50,171 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6241f0ff39574455987546cebd6b7303, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731397786880 2024-11-12T07:49:50,171 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2793cdcb05564658b799db51771bc43a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731397788217 2024-11-12T07:49:50,181 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:50,181 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/ea7b685ceca14632950140fc9da8385f is 50, key is test_row_0/C:col10/1731397788217/Put/seqid=0 2024-11-12T07:49:50,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742277_1453 (size=12595) 2024-11-12T07:49:50,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T07:49:50,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:50,243 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T07:49:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:50,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:50,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/f111fceea6b14ea898602beabecc3efb is 50, key is test_row_0/A:col10/1731397788905/Put/seqid=0 2024-11-12T07:49:50,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742278_1454 (size=12151) 2024-11-12T07:49:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:50,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:50,552 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/a4d676454926429ba808938e6765c92b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a4d676454926429ba808938e6765c92b 2024-11-12T07:49:50,556 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/B of e39e5d3040dc37fe65a005881116393f into a4d676454926429ba808938e6765c92b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:50,556 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:50,556 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/B, priority=13, startTime=1731397790129; duration=0sec 2024-11-12T07:49:50,556 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:50,556 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:B 2024-11-12T07:49:50,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397790559, max timestamp=1731397790584Num uniques 859; 2024-11-12T07:49:50,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397790559, max timestamp=1731397790584Num uniques 859; 2024-11-12T07:49:50,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397790559, max timestamp=1731397790584Num uniques 859; 2024-11-12T07:49:50,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:49:50,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:49:50,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:49:50,597 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/ea7b685ceca14632950140fc9da8385f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ea7b685ceca14632950140fc9da8385f 2024-11-12T07:49:50,601 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/C of e39e5d3040dc37fe65a005881116393f into ea7b685ceca14632950140fc9da8385f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:50,601 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:50,601 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/C, priority=13, startTime=1731397790129; duration=0sec 2024-11-12T07:49:50,601 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:50,601 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:C 2024-11-12T07:49:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T07:49:50,653 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/f111fceea6b14ea898602beabecc3efb 2024-11-12T07:49:50,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397850645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:50,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397850646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:50,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/f4c36c929e4e40cdbd9cedbfb6c2a767 is 50, key is test_row_0/B:col10/1731397788905/Put/seqid=0 2024-11-12T07:49:50,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742279_1455 (size=12151) 2024-11-12T07:49:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397850757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:50,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397850767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:50,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397850968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:50,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397850978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,063 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/f4c36c929e4e40cdbd9cedbfb6c2a767 2024-11-12T07:49:51,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/e45c0dc46e8e4223997547227ffefa6e is 50, key is test_row_0/C:col10/1731397788905/Put/seqid=0 2024-11-12T07:49:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397851068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742280_1456 (size=12151) 2024-11-12T07:49:51,072 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,074 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/e45c0dc46e8e4223997547227ffefa6e 2024-11-12T07:49:51,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/f111fceea6b14ea898602beabecc3efb as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f111fceea6b14ea898602beabecc3efb 2024-11-12T07:49:51,082 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f111fceea6b14ea898602beabecc3efb, entries=150, sequenceid=238, filesize=11.9 K 2024-11-12T07:49:51,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/f4c36c929e4e40cdbd9cedbfb6c2a767 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f4c36c929e4e40cdbd9cedbfb6c2a767 2024-11-12T07:49:51,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397851078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,089 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_0' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,091 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f4c36c929e4e40cdbd9cedbfb6c2a767, entries=150, sequenceid=238, filesize=11.9 K 2024-11-12T07:49:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/e45c0dc46e8e4223997547227ffefa6e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/e45c0dc46e8e4223997547227ffefa6e 2024-11-12T07:49:51,095 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/e45c0dc46e8e4223997547227ffefa6e, entries=150, sequenceid=238, filesize=11.9 K 2024-11-12T07:49:51,096 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for e39e5d3040dc37fe65a005881116393f in 852ms, sequenceid=238, compaction requested=false 2024-11-12T07:49:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:51,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-12T07:49:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-12T07:49:51,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-12T07:49:51,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6160 sec 2024-11-12T07:49:51,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.6220 sec 2024-11-12T07:49:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:51,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-12T07:49:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:51,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/25bd751f787e4d38a1d06cb71ce4b052 is 50, key is test_row_0/A:col10/1731397790610/Put/seqid=0 2024-11-12T07:49:51,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742281_1457 (size=14741) 2024-11-12T07:49:51,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397851309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397851310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397851417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397851427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T07:49:51,628 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:51,629 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-12T07:49:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T07:49:51,630 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:51,631 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:51,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:51,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397851637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397851637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/25bd751f787e4d38a1d06cb71ce4b052 2024-11-12T07:49:51,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/c516c429e9c0445bb3587b099780b681 is 50, key is test_row_0/B:col10/1731397790610/Put/seqid=0 2024-11-12T07:49:51,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742282_1458 (size=12301) 2024-11-12T07:49:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T07:49:51,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T07:49:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:51,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:51,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:51,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T07:49:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:51,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:51,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T07:49:51,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397851958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:51,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397851958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:51,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:52,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T07:49:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/c516c429e9c0445bb3587b099780b681 2024-11-12T07:49:52,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/231c85c481ef4d67a1f41cd14cb38a41 is 50, key is test_row_0/C:col10/1731397790610/Put/seqid=0 2024-11-12T07:49:52,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742283_1459 (size=12301) 2024-11-12T07:49:52,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T07:49:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T07:49:52,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T07:49:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:52,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397852468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397852477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:52,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/231c85c481ef4d67a1f41cd14cb38a41 2024-11-12T07:49:52,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/25bd751f787e4d38a1d06cb71ce4b052 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/25bd751f787e4d38a1d06cb71ce4b052 2024-11-12T07:49:52,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/25bd751f787e4d38a1d06cb71ce4b052, entries=200, sequenceid=278, filesize=14.4 K 2024-11-12T07:49:52,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/c516c429e9c0445bb3587b099780b681 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c516c429e9c0445bb3587b099780b681 2024-11-12T07:49:52,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c516c429e9c0445bb3587b099780b681, entries=150, sequenceid=278, filesize=12.0 K 2024-11-12T07:49:52,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/231c85c481ef4d67a1f41cd14cb38a41 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/231c85c481ef4d67a1f41cd14cb38a41 2024-11-12T07:49:52,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/231c85c481ef4d67a1f41cd14cb38a41, entries=150, sequenceid=278, filesize=12.0 K 2024-11-12T07:49:52,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for e39e5d3040dc37fe65a005881116393f in 1239ms, sequenceid=278, compaction requested=true 2024-11-12T07:49:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:52,524 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:52,524 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:52,525 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:52,525 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:52,525 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/A is initiating minor compaction (all files) 2024-11-12T07:49:52,525 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/B is initiating minor compaction (all files) 2024-11-12T07:49:52,525 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/B in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,525 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/A in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,526 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a4d676454926429ba808938e6765c92b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f4c36c929e4e40cdbd9cedbfb6c2a767, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c516c429e9c0445bb3587b099780b681] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=36.2 K 2024-11-12T07:49:52,526 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/48a82fbad87f4acea44332ffa1f9f6c3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f111fceea6b14ea898602beabecc3efb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/25bd751f787e4d38a1d06cb71ce4b052] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=38.6 K 2024-11-12T07:49:52,526 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d676454926429ba808938e6765c92b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731397788217 2024-11-12T07:49:52,526 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a82fbad87f4acea44332ffa1f9f6c3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731397788217 2024-11-12T07:49:52,526 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c36c929e4e40cdbd9cedbfb6c2a767, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731397788884 2024-11-12T07:49:52,526 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f111fceea6b14ea898602beabecc3efb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731397788884 2024-11-12T07:49:52,526 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c516c429e9c0445bb3587b099780b681, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731397790610 2024-11-12T07:49:52,526 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25bd751f787e4d38a1d06cb71ce4b052, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731397790610 2024-11-12T07:49:52,532 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:52,532 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#B#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:52,533 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/e7caae8270c34cc3af7d58591b7df44b is 50, key is test_row_0/A:col10/1731397790610/Put/seqid=0 2024-11-12T07:49:52,533 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/50a0fc93ab8e409895cf634906292a5e is 50, key is test_row_0/B:col10/1731397790610/Put/seqid=0 2024-11-12T07:49:52,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742285_1461 (size=12847) 2024-11-12T07:49:52,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742284_1460 (size=12847) 2024-11-12T07:49:52,543 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/e7caae8270c34cc3af7d58591b7df44b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e7caae8270c34cc3af7d58591b7df44b 2024-11-12T07:49:52,546 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/A of e39e5d3040dc37fe65a005881116393f into e7caae8270c34cc3af7d58591b7df44b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:52,547 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:52,547 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/A, priority=13, startTime=1731397792524; duration=0sec 2024-11-12T07:49:52,547 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:52,547 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:A 2024-11-12T07:49:52,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T07:49:52,547 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,547 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T07:49:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:52,547 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:52,548 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/C is initiating minor compaction (all files) 2024-11-12T07:49:52,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:52,548 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/C in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:52,548 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ea7b685ceca14632950140fc9da8385f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/e45c0dc46e8e4223997547227ffefa6e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/231c85c481ef4d67a1f41cd14cb38a41] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=36.2 K 2024-11-12T07:49:52,548 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea7b685ceca14632950140fc9da8385f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731397788217 2024-11-12T07:49:52,548 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e45c0dc46e8e4223997547227ffefa6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731397788884 2024-11-12T07:49:52,548 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 231c85c481ef4d67a1f41cd14cb38a41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731397790610 2024-11-12T07:49:52,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/b4616f4a552a43449852ecebe646f8ee is 50, key is test_row_0/A:col10/1731397791303/Put/seqid=0 2024-11-12T07:49:52,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742286_1462 (size=12301) 2024-11-12T07:49:52,556 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:52,556 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/fb7ae337d14d4f3eae7b7ad80870115a is 50, key is test_row_0/C:col10/1731397790610/Put/seqid=0 2024-11-12T07:49:52,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742287_1463 (size=12847) 2024-11-12T07:49:52,564 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/fb7ae337d14d4f3eae7b7ad80870115a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/fb7ae337d14d4f3eae7b7ad80870115a 2024-11-12T07:49:52,568 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/C of e39e5d3040dc37fe65a005881116393f into fb7ae337d14d4f3eae7b7ad80870115a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:52,568 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:52,568 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/C, priority=13, startTime=1731397792524; duration=0sec 2024-11-12T07:49:52,568 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:52,568 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:C 2024-11-12T07:49:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T07:49:52,944 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/50a0fc93ab8e409895cf634906292a5e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/50a0fc93ab8e409895cf634906292a5e 2024-11-12T07:49:52,948 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/B of e39e5d3040dc37fe65a005881116393f into 50a0fc93ab8e409895cf634906292a5e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:52,948 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:52,948 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/B, priority=13, startTime=1731397792524; duration=0sec 2024-11-12T07:49:52,948 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:52,948 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:B 2024-11-12T07:49:52,957 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/b4616f4a552a43449852ecebe646f8ee 2024-11-12T07:49:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/c4c6b5f95a34428c9714c3cda5a70626 is 50, key is test_row_0/B:col10/1731397791303/Put/seqid=0 2024-11-12T07:49:52,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742288_1464 (size=12301) 2024-11-12T07:49:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:53,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:53,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397793134, max timestamp=1731397793151Num uniques 859; 2024-11-12T07:49:53,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397793134, max timestamp=1731397793151Num uniques 859; 2024-11-12T07:49:53,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:49:53,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:49:53,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397793134, max timestamp=1731397793151Num uniques 859; 2024-11-12T07:49:53,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:49:53,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397793190, max timestamp=1731397793225Num uniques 1068; 2024-11-12T07:49:53,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397793190, max timestamp=1731397793225Num uniques 1068; 2024-11-12T07:49:53,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397793190, max timestamp=1731397793225Num uniques 1068; 2024-11-12T07:49:53,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T07:49:53,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T07:49:53,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-12T07:49:53,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397853291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731397793277, max timestamp=1731397793296Num uniques 1077; 2024-11-12T07:49:53,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T07:49:53,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731397793277, max timestamp=1731397793296Num uniques 1077; 2024-11-12T07:49:53,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731397793277, max timestamp=1731397793296Num uniques 1077; 2024-11-12T07:49:53,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-12T07:49:53,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T07:49:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,369 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/c4c6b5f95a34428c9714c3cda5a70626 2024-11-12T07:49:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/df038f67c686407ab54646344f906746 is 50, key is test_row_0/C:col10/1731397791303/Put/seqid=0 2024-11-12T07:49:53,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742289_1465 (size=12301) 2024-11-12T07:49:53,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397793388, max timestamp=1731397793405Num uniques 1086; 2024-11-12T07:49:53,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T07:49:53,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397793388, max timestamp=1731397793405Num uniques 1086; 2024-11-12T07:49:53,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T07:49:53,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397793388, max timestamp=1731397793405Num uniques 1036; 2024-11-12T07:49:53,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-12T07:49:53,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731397793483, max timestamp=1731397793500Num uniques 1045; 2024-11-12T07:49:53,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731397793483, max timestamp=1731397793500Num uniques 1045; 2024-11-12T07:49:53,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-12T07:49:53,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731397793483, max timestamp=1731397793500Num uniques 1045; 2024-11-12T07:49:53,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-12T07:49:53,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-11-12T07:49:53,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397853560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397853560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397853572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397853574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397853678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397853678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397853687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397853697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,778 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/df038f67c686407ab54646344f906746 2024-11-12T07:49:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T07:49:53,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/b4616f4a552a43449852ecebe646f8ee as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/b4616f4a552a43449852ecebe646f8ee 2024-11-12T07:49:53,784 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/b4616f4a552a43449852ecebe646f8ee, entries=150, sequenceid=287, filesize=12.0 K 2024-11-12T07:49:53,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/c4c6b5f95a34428c9714c3cda5a70626 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c4c6b5f95a34428c9714c3cda5a70626 2024-11-12T07:49:53,786 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c4c6b5f95a34428c9714c3cda5a70626, entries=150, sequenceid=287, filesize=12.0 K 2024-11-12T07:49:53,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/df038f67c686407ab54646344f906746 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/df038f67c686407ab54646344f906746 2024-11-12T07:49:53,789 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/df038f67c686407ab54646344f906746, entries=150, sequenceid=287, filesize=12.0 K 2024-11-12T07:49:53,790 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for e39e5d3040dc37fe65a005881116393f in 1243ms, sequenceid=287, compaction requested=false 2024-11-12T07:49:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-12T07:49:53,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-12T07:49:53,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-12T07:49:53,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-11-12T07:49:53,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1640 sec 2024-11-12T07:49:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:53,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-12T07:49:53,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:53,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:53,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:53,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:53,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:53,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:49:53,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/67607c9a234d48f48e2fdfbbce137fa5 is 50, key is test_row_0/A:col10/1731397793515/Put/seqid=0 2024-11-12T07:49:53,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742290_1466 (size=14741) 2024-11-12T07:49:53,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397853908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397853940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397853940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397853940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397854058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397854058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397854059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397854248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397854278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397854278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397854278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/67607c9a234d48f48e2fdfbbce137fa5 2024-11-12T07:49:54,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/165bfe2a35dd41a1bf193a24b0b4a4b1 is 50, key is test_row_0/B:col10/1731397793515/Put/seqid=0 2024-11-12T07:49:54,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742291_1467 (size=12301) 2024-11-12T07:49:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397854587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397854598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397854598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:54,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/165bfe2a35dd41a1bf193a24b0b4a4b1 2024-11-12T07:49:54,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/928cf3e4c6e6487497f0ab7fb4da7b34 is 50, key is test_row_0/C:col10/1731397793515/Put/seqid=0 2024-11-12T07:49:54,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742292_1468 (size=12301) 2024-11-12T07:49:54,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397854768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:54,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397855108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397855108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397855108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:55,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/928cf3e4c6e6487497f0ab7fb4da7b34 2024-11-12T07:49:55,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/67607c9a234d48f48e2fdfbbce137fa5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/67607c9a234d48f48e2fdfbbce137fa5 2024-11-12T07:49:55,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/67607c9a234d48f48e2fdfbbce137fa5, entries=200, sequenceid=395, filesize=14.4 K 2024-11-12T07:49:55,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/165bfe2a35dd41a1bf193a24b0b4a4b1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/165bfe2a35dd41a1bf193a24b0b4a4b1 2024-11-12T07:49:55,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/165bfe2a35dd41a1bf193a24b0b4a4b1, entries=150, sequenceid=395, filesize=12.0 K 2024-11-12T07:49:55,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/928cf3e4c6e6487497f0ab7fb4da7b34 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/928cf3e4c6e6487497f0ab7fb4da7b34 2024-11-12T07:49:55,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/928cf3e4c6e6487497f0ab7fb4da7b34, entries=150, sequenceid=395, filesize=12.0 K 2024-11-12T07:49:55,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for e39e5d3040dc37fe65a005881116393f in 1244ms, sequenceid=395, compaction requested=true 2024-11-12T07:49:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:55,149 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:55,149 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:55,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:55,149 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:55,150 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/A is initiating minor compaction (all files) 2024-11-12T07:49:55,150 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:55,150 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/A in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:55,150 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/B is initiating minor compaction (all files) 2024-11-12T07:49:55,150 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/B in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:55,150 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e7caae8270c34cc3af7d58591b7df44b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/b4616f4a552a43449852ecebe646f8ee, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/67607c9a234d48f48e2fdfbbce137fa5] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=39.0 K 2024-11-12T07:49:55,150 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/50a0fc93ab8e409895cf634906292a5e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c4c6b5f95a34428c9714c3cda5a70626, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/165bfe2a35dd41a1bf193a24b0b4a4b1] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=36.6 K 2024-11-12T07:49:55,150 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7caae8270c34cc3af7d58591b7df44b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731397790610 2024-11-12T07:49:55,150 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a0fc93ab8e409895cf634906292a5e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731397790610 2024-11-12T07:49:55,150 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c6b5f95a34428c9714c3cda5a70626, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731397791303 2024-11-12T07:49:55,150 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4616f4a552a43449852ecebe646f8ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731397791303 2024-11-12T07:49:55,150 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 165bfe2a35dd41a1bf193a24b0b4a4b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731397793515 2024-11-12T07:49:55,151 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67607c9a234d48f48e2fdfbbce137fa5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731397793515 2024-11-12T07:49:55,156 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:55,156 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#A#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:55,157 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/f7e6a5bc505a4a8d9d0d0d523c810628 is 50, key is test_row_0/B:col10/1731397793515/Put/seqid=0 2024-11-12T07:49:55,157 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/5579352a20b3462bb8ca3fa28a84922d is 50, key is test_row_0/A:col10/1731397793515/Put/seqid=0 2024-11-12T07:49:55,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742293_1469 (size=12949) 2024-11-12T07:49:55,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742294_1470 (size=12949) 2024-11-12T07:49:55,165 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/5579352a20b3462bb8ca3fa28a84922d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5579352a20b3462bb8ca3fa28a84922d 2024-11-12T07:49:55,169 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/A of e39e5d3040dc37fe65a005881116393f into 5579352a20b3462bb8ca3fa28a84922d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:55,169 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:55,169 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/A, priority=13, startTime=1731397795149; duration=0sec 2024-11-12T07:49:55,169 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:55,169 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:A 2024-11-12T07:49:55,169 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:55,170 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:55,170 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/C is initiating minor compaction (all files) 2024-11-12T07:49:55,170 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/C in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:55,170 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/fb7ae337d14d4f3eae7b7ad80870115a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/df038f67c686407ab54646344f906746, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/928cf3e4c6e6487497f0ab7fb4da7b34] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=36.6 K 2024-11-12T07:49:55,170 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb7ae337d14d4f3eae7b7ad80870115a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731397790610 2024-11-12T07:49:55,170 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting df038f67c686407ab54646344f906746, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731397791303 2024-11-12T07:49:55,171 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 928cf3e4c6e6487497f0ab7fb4da7b34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731397793515 2024-11-12T07:49:55,176 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#C#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:55,177 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/c23030d5cc6c46c28069e7a6dacd620a is 50, key is test_row_0/C:col10/1731397793515/Put/seqid=0 2024-11-12T07:49:55,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742295_1471 (size=12949) 2024-11-12T07:49:55,568 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/f7e6a5bc505a4a8d9d0d0d523c810628 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f7e6a5bc505a4a8d9d0d0d523c810628 2024-11-12T07:49:55,571 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/B of e39e5d3040dc37fe65a005881116393f into f7e6a5bc505a4a8d9d0d0d523c810628(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:55,572 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:55,572 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/B, priority=13, startTime=1731397795149; duration=0sec 2024-11-12T07:49:55,572 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:55,572 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:B 2024-11-12T07:49:55,584 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/c23030d5cc6c46c28069e7a6dacd620a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/c23030d5cc6c46c28069e7a6dacd620a 2024-11-12T07:49:55,588 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/C of e39e5d3040dc37fe65a005881116393f into c23030d5cc6c46c28069e7a6dacd620a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:55,588 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:55,588 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/C, priority=13, startTime=1731397795149; duration=0sec 2024-11-12T07:49:55,588 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:55,588 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:C 2024-11-12T07:49:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:55,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:49:55,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:55,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:55,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:55,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:55,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:55,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T07:49:55,798 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:49:55,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:49:55,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/e93eebd76afa46f6bc32c6416834e781 is 50, key is test_row_0/A:col10/1731397795792/Put/seqid=0 2024-11-12T07:49:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-12T07:49:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T07:49:55,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:49:55,802 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:49:55,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:49:55,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742296_1472 (size=17181) 2024-11-12T07:49:55,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397795846, max timestamp=1731397795888Num uniques 859; 2024-11-12T07:49:55,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397795846, max timestamp=1731397795888Num uniques 859; 2024-11-12T07:49:55,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:55,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397795846, max timestamp=1731397795888Num uniques 859; 2024-11-12T07:49:55,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:55,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:49:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T07:49:55,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:55,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:55,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397856001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:56,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:56,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:56,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T07:49:56,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397856118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:56,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:56,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397856118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:56,122 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:56,123 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:56,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:56,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397856118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:56,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397856128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:56,141 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:56,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/e93eebd76afa46f6bc32c6416834e781 2024-11-12T07:49:56,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/d8e6ae9610924456abe6b40c1610aee1 is 50, key is test_row_0/B:col10/1731397795792/Put/seqid=0 2024-11-12T07:49:56,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742297_1473 (size=12301) 2024-11-12T07:49:56,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397856328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:56,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:56,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:56,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T07:49:56,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:56,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:56,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/d8e6ae9610924456abe6b40c1610aee1 2024-11-12T07:49:56,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/d77730648bed499881c48ec689edf4f3 is 50, key is test_row_0/C:col10/1731397795792/Put/seqid=0 2024-11-12T07:49:56,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742298_1474 (size=12301) 2024-11-12T07:49:56,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397856648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:56,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:56,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:56,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:56,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:56,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:56,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T07:49:57,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/d77730648bed499881c48ec689edf4f3 2024-11-12T07:49:57,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/e93eebd76afa46f6bc32c6416834e781 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e93eebd76afa46f6bc32c6416834e781 2024-11-12T07:49:57,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e93eebd76afa46f6bc32c6416834e781, entries=250, sequenceid=408, filesize=16.8 K 2024-11-12T07:49:57,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/d8e6ae9610924456abe6b40c1610aee1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/d8e6ae9610924456abe6b40c1610aee1 2024-11-12T07:49:57,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/d8e6ae9610924456abe6b40c1610aee1, entries=150, sequenceid=408, filesize=12.0 K 2024-11-12T07:49:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/d77730648bed499881c48ec689edf4f3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d77730648bed499881c48ec689edf4f3 2024-11-12T07:49:57,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d77730648bed499881c48ec689edf4f3, entries=150, sequenceid=408, filesize=12.0 K 2024-11-12T07:49:57,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e39e5d3040dc37fe65a005881116393f in 1241ms, sequenceid=408, compaction requested=false 2024-11-12T07:49:57,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:57,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T07:49:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:57,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:49:57,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/eafb2aeac71f49a89b86cb8a9827f825 is 50, key is test_row_0/A:col10/1731397795999/Put/seqid=0 2024-11-12T07:49:57,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742299_1475 (size=14741) 2024-11-12T07:49:57,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397857205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:57,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52306 deadline: 1731397857237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:57,244 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18335 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:57,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397857318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:57,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:57,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:57,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:57,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:57,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397857528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:57,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/eafb2aeac71f49a89b86cb8a9827f825 2024-11-12T07:49:57,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/59dcd24d991c4b14a379295a0c2b8e53 is 50, key is test_row_0/B:col10/1731397795999/Put/seqid=0 2024-11-12T07:49:57,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742300_1476 (size=12301) 2024-11-12T07:49:57,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397857848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:57,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:57,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:57,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T07:49:57,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/59dcd24d991c4b14a379295a0c2b8e53 2024-11-12T07:49:57,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/f5f9cca95caa4663a5950dfb8746d326 is 50, key is test_row_0/C:col10/1731397795999/Put/seqid=0 2024-11-12T07:49:57,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742301_1477 (size=12301) 2024-11-12T07:49:58,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:58,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:58,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52256 deadline: 1731397858137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:58,142 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52284 deadline: 1731397858147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:58,154 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:58,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:58,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52332 deadline: 1731397858158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:58,166 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in e39e5d3040dc37fe65a005881116393f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:49:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:58,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:58,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:58,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:49:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52314 deadline: 1731397858358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:49:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 , the old value is region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:49:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., hostname=f9fc96254ef8,44299,1731397650797, seqNum=2 because the exception is null or not the one we care about 2024-11-12T07:49:58,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/f5f9cca95caa4663a5950dfb8746d326 2024-11-12T07:49:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/eafb2aeac71f49a89b86cb8a9827f825 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eafb2aeac71f49a89b86cb8a9827f825 2024-11-12T07:49:58,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eafb2aeac71f49a89b86cb8a9827f825, entries=200, sequenceid=446, filesize=14.4 K 2024-11-12T07:49:58,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/59dcd24d991c4b14a379295a0c2b8e53 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/59dcd24d991c4b14a379295a0c2b8e53 2024-11-12T07:49:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:49:58,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/59dcd24d991c4b14a379295a0c2b8e53, entries=150, sequenceid=446, filesize=12.0 K 2024-11-12T07:49:58,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/f5f9cca95caa4663a5950dfb8746d326 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f5f9cca95caa4663a5950dfb8746d326 2024-11-12T07:49:58,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f5f9cca95caa4663a5950dfb8746d326, entries=150, sequenceid=446, filesize=12.0 K 2024-11-12T07:49:58,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for e39e5d3040dc37fe65a005881116393f in 1239ms, sequenceid=446, compaction requested=true 2024-11-12T07:49:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:49:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:58,404 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:49:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:58,404 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e39e5d3040dc37fe65a005881116393f:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:49:58,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:58,405 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:58,405 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44871 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:58,405 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/A is initiating minor compaction (all files) 2024-11-12T07:49:58,405 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/B is initiating minor compaction (all files) 2024-11-12T07:49:58,405 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/B in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,405 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/A in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,405 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5579352a20b3462bb8ca3fa28a84922d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e93eebd76afa46f6bc32c6416834e781, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eafb2aeac71f49a89b86cb8a9827f825] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=43.8 K 2024-11-12T07:49:58,405 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f7e6a5bc505a4a8d9d0d0d523c810628, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/d8e6ae9610924456abe6b40c1610aee1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/59dcd24d991c4b14a379295a0c2b8e53] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=36.7 K 2024-11-12T07:49:58,405 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e6a5bc505a4a8d9d0d0d523c810628, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731397793515 2024-11-12T07:49:58,405 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5579352a20b3462bb8ca3fa28a84922d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731397793515 2024-11-12T07:49:58,406 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e6ae9610924456abe6b40c1610aee1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731397793910 2024-11-12T07:49:58,406 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e93eebd76afa46f6bc32c6416834e781, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731397793910 2024-11-12T07:49:58,406 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 59dcd24d991c4b14a379295a0c2b8e53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731397795975 2024-11-12T07:49:58,406 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting eafb2aeac71f49a89b86cb8a9827f825, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731397795975 2024-11-12T07:49:58,412 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:58,412 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:58,412 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/09c4e6ca7098488281d4b8c0f1b59f00 is 50, key is test_row_0/B:col10/1731397795999/Put/seqid=0 2024-11-12T07:49:58,412 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/0594c81f39ea4e03b9e663f4bde00aec is 50, key is test_row_0/A:col10/1731397795999/Put/seqid=0 2024-11-12T07:49:58,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742303_1479 (size=13051) 2024-11-12T07:49:58,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742302_1478 (size=13051) 2024-11-12T07:49:58,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T07:49:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,550 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T07:49:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:49:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:49:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:49:58,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:49:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/65118cf11ba34c5da6cc43286fc6fe67 is 50, key is test_row_0/A:col10/1731397797187/Put/seqid=0 2024-11-12T07:49:58,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742304_1480 (size=12301) 2024-11-12T07:49:58,557 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/65118cf11ba34c5da6cc43286fc6fe67 2024-11-12T07:49:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/61624f05c8c24450bb29543fbd94bc2c is 50, key is test_row_0/B:col10/1731397797187/Put/seqid=0 2024-11-12T07:49:58,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742305_1481 (size=12301) 2024-11-12T07:49:58,820 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/09c4e6ca7098488281d4b8c0f1b59f00 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/09c4e6ca7098488281d4b8c0f1b59f00 2024-11-12T07:49:58,823 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/0594c81f39ea4e03b9e663f4bde00aec as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/0594c81f39ea4e03b9e663f4bde00aec 2024-11-12T07:49:58,823 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/B of e39e5d3040dc37fe65a005881116393f into 09c4e6ca7098488281d4b8c0f1b59f00(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:58,823 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:58,824 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/B, priority=13, startTime=1731397798404; duration=0sec 2024-11-12T07:49:58,824 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:49:58,824 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:B 2024-11-12T07:49:58,824 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:49:58,827 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:49:58,827 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): e39e5d3040dc37fe65a005881116393f/C is initiating minor compaction (all files) 2024-11-12T07:49:58,827 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e39e5d3040dc37fe65a005881116393f/C in TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:58,827 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/c23030d5cc6c46c28069e7a6dacd620a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d77730648bed499881c48ec689edf4f3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f5f9cca95caa4663a5950dfb8746d326] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp, totalSize=36.7 K 2024-11-12T07:49:58,828 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting c23030d5cc6c46c28069e7a6dacd620a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731397793515 2024-11-12T07:49:58,828 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d77730648bed499881c48ec689edf4f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731397793910 2024-11-12T07:49:58,828 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f5f9cca95caa4663a5950dfb8746d326, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731397795975 2024-11-12T07:49:58,831 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/A of e39e5d3040dc37fe65a005881116393f into 0594c81f39ea4e03b9e663f4bde00aec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:58,831 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:58,831 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/A, priority=13, startTime=1731397798404; duration=0sec 2024-11-12T07:49:58,831 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:58,831 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:A 2024-11-12T07:49:58,844 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e39e5d3040dc37fe65a005881116393f#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:49:58,845 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/214b1ad440424467a58979ef28f0e34d is 50, key is test_row_0/C:col10/1731397795999/Put/seqid=0 2024-11-12T07:49:58,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742306_1482 (size=13051) 2024-11-12T07:49:58,849 INFO [Thread-1874 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1874. 2024-11-12T07:49:58,849 DEBUG [Thread-1874 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:58,849 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,849 INFO [Thread-1872 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1872. 2024-11-12T07:49:58,849 DEBUG [Thread-1872 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:58,849 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,849 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:58,849 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:58,852 INFO [Thread-1876 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1876. 2024-11-12T07:49:58,853 DEBUG [Thread-1876 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:58,853 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,853 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:58,854 INFO [Thread-1878 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1878. 2024-11-12T07:49:58,854 DEBUG [Thread-1878 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:58,854 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,854 INFO [Thread-1870 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1870. 2024-11-12T07:49:58,854 DEBUG [Thread-1870 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:58,854 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,854 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:58,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:58,854 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:58,973 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/61624f05c8c24450bb29543fbd94bc2c 2024-11-12T07:49:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/f4ed906a27a140508624777996ee0343 is 50, key is test_row_0/C:col10/1731397797187/Put/seqid=0 2024-11-12T07:49:58,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742307_1483 (size=12301) 2024-11-12T07:49:59,130 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:49:59,259 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/214b1ad440424467a58979ef28f0e34d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/214b1ad440424467a58979ef28f0e34d 2024-11-12T07:49:59,267 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e39e5d3040dc37fe65a005881116393f/C of e39e5d3040dc37fe65a005881116393f into 214b1ad440424467a58979ef28f0e34d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:49:59,267 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:59,267 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., storeName=e39e5d3040dc37fe65a005881116393f/C, priority=13, startTime=1731397798404; duration=0sec 2024-11-12T07:49:59,267 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:49:59,267 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e39e5d3040dc37fe65a005881116393f:C 2024-11-12T07:49:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on e39e5d3040dc37fe65a005881116393f 2024-11-12T07:49:59,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. as already flushing 2024-11-12T07:49:59,384 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-11-12T07:49:59,384 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:49:59,384 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:59,384 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:49:59,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:49:59,395 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/f4ed906a27a140508624777996ee0343 2024-11-12T07:49:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/65118cf11ba34c5da6cc43286fc6fe67 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/65118cf11ba34c5da6cc43286fc6fe67 2024-11-12T07:49:59,406 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/65118cf11ba34c5da6cc43286fc6fe67, entries=150, sequenceid=455, filesize=12.0 K 2024-11-12T07:49:59,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/61624f05c8c24450bb29543fbd94bc2c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/61624f05c8c24450bb29543fbd94bc2c 2024-11-12T07:49:59,413 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/61624f05c8c24450bb29543fbd94bc2c, entries=150, sequenceid=455, filesize=12.0 K 2024-11-12T07:49:59,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/f4ed906a27a140508624777996ee0343 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f4ed906a27a140508624777996ee0343 2024-11-12T07:49:59,418 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f4ed906a27a140508624777996ee0343, entries=150, sequenceid=455, filesize=12.0 K 2024-11-12T07:49:59,419 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for e39e5d3040dc37fe65a005881116393f in 869ms, sequenceid=455, compaction requested=false 2024-11-12T07:49:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e39e5d3040dc37fe65a005881116393f: 2024-11-12T07:49:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:49:59,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-12T07:49:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-12T07:49:59,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-12T07:49:59,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6180 sec 2024-11-12T07:49:59,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.6230 sec 2024-11-12T07:49:59,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T07:49:59,970 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:02,182 INFO [Thread-1867 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1867. 2024-11-12T07:50:02,183 DEBUG [Thread-1867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:02,183 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:02,183 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:02,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:02,201 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-11-12T07:50:02,201 INFO [Thread-1865 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1865. 2024-11-12T07:50:02,201 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:02,201 DEBUG [Thread-1865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:02,201 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:02,201 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:02,201 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:02,201 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:02,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:02,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:07,353 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-11-12T07:50:07,353 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:07,353 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:07,353 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:07,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:07,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T07:50:07,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-12T07:50:07,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-12T07:50:07,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-12T07:50:07,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1291 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3873 rows 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1295 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3885 rows 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1290 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3866 rows 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1287 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3859 rows 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1287 2024-11-12T07:50:07,355 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3859 rows 2024-11-12T07:50:07,355 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:50:07,356 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:07,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:07,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:07,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:07,358 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T07:50:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T07:50:07,362 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397807362"}]},"ts":"1731397807362"} 2024-11-12T07:50:07,364 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T07:50:07,364 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T07:50:07,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:50:07,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, UNASSIGN}] 2024-11-12T07:50:07,368 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, UNASSIGN 2024-11-12T07:50:07,369 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e39e5d3040dc37fe65a005881116393f, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:07,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, UNASSIGN because future has completed 2024-11-12T07:50:07,371 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:50:07,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:50:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T07:50:07,523 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close e39e5d3040dc37fe65a005881116393f 2024-11-12T07:50:07,523 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:50:07,524 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing e39e5d3040dc37fe65a005881116393f, disabling compactions & flushes 2024-11-12T07:50:07,524 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:50:07,524 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:50:07,524 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. after waiting 0 ms 2024-11-12T07:50:07,524 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:50:07,524 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing e39e5d3040dc37fe65a005881116393f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T07:50:07,524 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=A 2024-11-12T07:50:07,524 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:07,525 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=B 2024-11-12T07:50:07,525 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:07,525 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e39e5d3040dc37fe65a005881116393f, store=C 2024-11-12T07:50:07,525 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:07,531 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/91b6ff7568ea40d8b168caf87bb15f69 is 50, key is test_row_1/A:col10/1731397802179/Put/seqid=0 2024-11-12T07:50:07,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742308_1484 (size=9857) 2024-11-12T07:50:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T07:50:07,938 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/91b6ff7568ea40d8b168caf87bb15f69 2024-11-12T07:50:07,953 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/3182e4cf2a2b4fd092a7d884fc09a054 is 50, key is test_row_1/B:col10/1731397802179/Put/seqid=0 2024-11-12T07:50:07,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742309_1485 (size=9857) 2024-11-12T07:50:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T07:50:08,357 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/3182e4cf2a2b4fd092a7d884fc09a054 2024-11-12T07:50:08,363 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/5d25555906f242c98fad651cad836e23 is 50, key is test_row_1/C:col10/1731397802179/Put/seqid=0 2024-11-12T07:50:08,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742310_1486 (size=9857) 2024-11-12T07:50:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T07:50:08,767 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/5d25555906f242c98fad651cad836e23 2024-11-12T07:50:08,774 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/A/91b6ff7568ea40d8b168caf87bb15f69 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/91b6ff7568ea40d8b168caf87bb15f69 2024-11-12T07:50:08,781 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/91b6ff7568ea40d8b168caf87bb15f69, entries=100, sequenceid=466, filesize=9.6 K 2024-11-12T07:50:08,782 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/B/3182e4cf2a2b4fd092a7d884fc09a054 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/3182e4cf2a2b4fd092a7d884fc09a054 2024-11-12T07:50:08,788 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/3182e4cf2a2b4fd092a7d884fc09a054, entries=100, sequenceid=466, filesize=9.6 K 2024-11-12T07:50:08,788 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/.tmp/C/5d25555906f242c98fad651cad836e23 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5d25555906f242c98fad651cad836e23 2024-11-12T07:50:08,793 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5d25555906f242c98fad651cad836e23, entries=100, sequenceid=466, filesize=9.6 K 2024-11-12T07:50:08,794 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e39e5d3040dc37fe65a005881116393f in 1270ms, sequenceid=466, compaction requested=true 2024-11-12T07:50:08,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/56555bd915524be291972b0bdcff4a8e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9fee473bb4754cf38ae932e4645e0b8f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5c9d49ed4a3b420db15b6e9dcfbc66e9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/da2eaf9278f341d6b6894e5bf3dcc944, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/435bb31bc22f4ba9aef20271927db115, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/c861983b1ae649eb990aa9938d931806, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/7f9a0149a4ab471eae1aff9fedfeac79, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/21dffdb35e364f22b97a2c07f2b40a22, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9ea854a72472445f964a7b84fce6e668, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/69a88b6815d44073bde3cb3612cecc79, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f8e7be3452f14fc8b9d5c6bea94789ab, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eb0b5d1c8790472ab776135f97d9900f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/215b2baccce141f78f6e60495dc55222, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/48a82fbad87f4acea44332ffa1f9f6c3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f111fceea6b14ea898602beabecc3efb, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/25bd751f787e4d38a1d06cb71ce4b052, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e7caae8270c34cc3af7d58591b7df44b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/b4616f4a552a43449852ecebe646f8ee, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/67607c9a234d48f48e2fdfbbce137fa5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5579352a20b3462bb8ca3fa28a84922d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e93eebd76afa46f6bc32c6416834e781, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eafb2aeac71f49a89b86cb8a9827f825] to archive 2024-11-12T07:50:08,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:50:08,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/56555bd915524be291972b0bdcff4a8e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/56555bd915524be291972b0bdcff4a8e 2024-11-12T07:50:08,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9fee473bb4754cf38ae932e4645e0b8f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9fee473bb4754cf38ae932e4645e0b8f 2024-11-12T07:50:08,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5c9d49ed4a3b420db15b6e9dcfbc66e9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5c9d49ed4a3b420db15b6e9dcfbc66e9 2024-11-12T07:50:08,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/da2eaf9278f341d6b6894e5bf3dcc944 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/da2eaf9278f341d6b6894e5bf3dcc944 2024-11-12T07:50:08,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/435bb31bc22f4ba9aef20271927db115 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/435bb31bc22f4ba9aef20271927db115 2024-11-12T07:50:08,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/c861983b1ae649eb990aa9938d931806 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/c861983b1ae649eb990aa9938d931806 2024-11-12T07:50:08,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/7f9a0149a4ab471eae1aff9fedfeac79 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/7f9a0149a4ab471eae1aff9fedfeac79 2024-11-12T07:50:08,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/21dffdb35e364f22b97a2c07f2b40a22 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/21dffdb35e364f22b97a2c07f2b40a22 2024-11-12T07:50:08,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9ea854a72472445f964a7b84fce6e668 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/9ea854a72472445f964a7b84fce6e668 2024-11-12T07:50:08,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/69a88b6815d44073bde3cb3612cecc79 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/69a88b6815d44073bde3cb3612cecc79 2024-11-12T07:50:08,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f8e7be3452f14fc8b9d5c6bea94789ab to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f8e7be3452f14fc8b9d5c6bea94789ab 2024-11-12T07:50:08,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eb0b5d1c8790472ab776135f97d9900f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eb0b5d1c8790472ab776135f97d9900f 2024-11-12T07:50:08,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/215b2baccce141f78f6e60495dc55222 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/215b2baccce141f78f6e60495dc55222 2024-11-12T07:50:08,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/48a82fbad87f4acea44332ffa1f9f6c3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/48a82fbad87f4acea44332ffa1f9f6c3 2024-11-12T07:50:08,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f111fceea6b14ea898602beabecc3efb to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/f111fceea6b14ea898602beabecc3efb 2024-11-12T07:50:08,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/25bd751f787e4d38a1d06cb71ce4b052 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/25bd751f787e4d38a1d06cb71ce4b052 2024-11-12T07:50:08,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e7caae8270c34cc3af7d58591b7df44b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e7caae8270c34cc3af7d58591b7df44b 2024-11-12T07:50:08,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/b4616f4a552a43449852ecebe646f8ee to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/b4616f4a552a43449852ecebe646f8ee 2024-11-12T07:50:08,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/67607c9a234d48f48e2fdfbbce137fa5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/67607c9a234d48f48e2fdfbbce137fa5 2024-11-12T07:50:08,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5579352a20b3462bb8ca3fa28a84922d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/5579352a20b3462bb8ca3fa28a84922d 2024-11-12T07:50:08,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e93eebd76afa46f6bc32c6416834e781 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/e93eebd76afa46f6bc32c6416834e781 2024-11-12T07:50:08,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eafb2aeac71f49a89b86cb8a9827f825 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/eafb2aeac71f49a89b86cb8a9827f825 2024-11-12T07:50:08,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7dea781080104eeba3a2bfa8d408334d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/ffab0233f1414d3686ce3c8e55801d1a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/715a801b14f84d49bf9a4df8f7931678, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7e9daaa9578e49c9a6aaa90173b9246a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5e0cdfde15a4d94bce1901ad7b6f14b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/813a71c4d0474130bbb493c9bbda14e1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5cab41903354d649b21adb92d2228dd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5d1282b576ef4041a649887cfbb0c7a5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/1b24a1e693f24709b7b28294cf2a0a6c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/dfe3ffb44e7a4882be95a4929162574b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/56339212ff8544738ee3dacb76aa58b0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/95a42063732c48f3ac0543216e72bfbd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a4d676454926429ba808938e6765c92b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5a5aec526c3043bdb9805cfc415c50e8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f4c36c929e4e40cdbd9cedbfb6c2a767, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/50a0fc93ab8e409895cf634906292a5e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c516c429e9c0445bb3587b099780b681, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c4c6b5f95a34428c9714c3cda5a70626, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f7e6a5bc505a4a8d9d0d0d523c810628, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/165bfe2a35dd41a1bf193a24b0b4a4b1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/d8e6ae9610924456abe6b40c1610aee1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/59dcd24d991c4b14a379295a0c2b8e53] to archive 2024-11-12T07:50:08,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:50:08,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7dea781080104eeba3a2bfa8d408334d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7dea781080104eeba3a2bfa8d408334d 2024-11-12T07:50:08,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/ffab0233f1414d3686ce3c8e55801d1a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/ffab0233f1414d3686ce3c8e55801d1a 2024-11-12T07:50:08,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/715a801b14f84d49bf9a4df8f7931678 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/715a801b14f84d49bf9a4df8f7931678 2024-11-12T07:50:08,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7e9daaa9578e49c9a6aaa90173b9246a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/7e9daaa9578e49c9a6aaa90173b9246a 2024-11-12T07:50:08,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5e0cdfde15a4d94bce1901ad7b6f14b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5e0cdfde15a4d94bce1901ad7b6f14b 2024-11-12T07:50:08,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/813a71c4d0474130bbb493c9bbda14e1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/813a71c4d0474130bbb493c9bbda14e1 2024-11-12T07:50:08,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5cab41903354d649b21adb92d2228dd to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a5cab41903354d649b21adb92d2228dd 2024-11-12T07:50:08,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5d1282b576ef4041a649887cfbb0c7a5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5d1282b576ef4041a649887cfbb0c7a5 2024-11-12T07:50:08,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/1b24a1e693f24709b7b28294cf2a0a6c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/1b24a1e693f24709b7b28294cf2a0a6c 2024-11-12T07:50:08,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/dfe3ffb44e7a4882be95a4929162574b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/dfe3ffb44e7a4882be95a4929162574b 2024-11-12T07:50:08,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/56339212ff8544738ee3dacb76aa58b0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/56339212ff8544738ee3dacb76aa58b0 2024-11-12T07:50:08,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/95a42063732c48f3ac0543216e72bfbd to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/95a42063732c48f3ac0543216e72bfbd 2024-11-12T07:50:08,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a4d676454926429ba808938e6765c92b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/a4d676454926429ba808938e6765c92b 2024-11-12T07:50:08,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5a5aec526c3043bdb9805cfc415c50e8 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/5a5aec526c3043bdb9805cfc415c50e8 2024-11-12T07:50:08,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f4c36c929e4e40cdbd9cedbfb6c2a767 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f4c36c929e4e40cdbd9cedbfb6c2a767 2024-11-12T07:50:08,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/50a0fc93ab8e409895cf634906292a5e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/50a0fc93ab8e409895cf634906292a5e 2024-11-12T07:50:08,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c516c429e9c0445bb3587b099780b681 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c516c429e9c0445bb3587b099780b681 2024-11-12T07:50:08,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c4c6b5f95a34428c9714c3cda5a70626 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/c4c6b5f95a34428c9714c3cda5a70626 2024-11-12T07:50:08,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f7e6a5bc505a4a8d9d0d0d523c810628 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/f7e6a5bc505a4a8d9d0d0d523c810628 2024-11-12T07:50:08,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/165bfe2a35dd41a1bf193a24b0b4a4b1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/165bfe2a35dd41a1bf193a24b0b4a4b1 2024-11-12T07:50:08,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/d8e6ae9610924456abe6b40c1610aee1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/d8e6ae9610924456abe6b40c1610aee1 2024-11-12T07:50:08,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/59dcd24d991c4b14a379295a0c2b8e53 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/59dcd24d991c4b14a379295a0c2b8e53 2024-11-12T07:50:08,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d0d26fe0b28044bab782433b974555a3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/cb956afc9371453a95264d73505d1c43, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6874181875104cffa5a438026a2b88f7, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/08bb155eb8fb44d3abbf42524b535f34, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/86a1919d9e0249edbd787175c2efc354, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/99e64e3024bd4460be504330f146d6ab, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ce8342ee5abd4486b425666cd9f317c3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2014e53fdc38453ab770013b4933461d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5f1d7e98b23d4ad3877ef48079ae4de0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/21a32f061baf4081947893425d93db66, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/643a09363c954387a223c4820a4f063d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6241f0ff39574455987546cebd6b7303, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ea7b685ceca14632950140fc9da8385f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2793cdcb05564658b799db51771bc43a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/e45c0dc46e8e4223997547227ffefa6e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/fb7ae337d14d4f3eae7b7ad80870115a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/231c85c481ef4d67a1f41cd14cb38a41, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/df038f67c686407ab54646344f906746, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/c23030d5cc6c46c28069e7a6dacd620a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/928cf3e4c6e6487497f0ab7fb4da7b34, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d77730648bed499881c48ec689edf4f3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f5f9cca95caa4663a5950dfb8746d326] to archive 2024-11-12T07:50:08,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:50:08,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d0d26fe0b28044bab782433b974555a3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d0d26fe0b28044bab782433b974555a3 2024-11-12T07:50:08,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/cb956afc9371453a95264d73505d1c43 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/cb956afc9371453a95264d73505d1c43 2024-11-12T07:50:08,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6874181875104cffa5a438026a2b88f7 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6874181875104cffa5a438026a2b88f7 2024-11-12T07:50:08,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/08bb155eb8fb44d3abbf42524b535f34 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/08bb155eb8fb44d3abbf42524b535f34 2024-11-12T07:50:08,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/86a1919d9e0249edbd787175c2efc354 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/86a1919d9e0249edbd787175c2efc354 2024-11-12T07:50:08,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/99e64e3024bd4460be504330f146d6ab to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/99e64e3024bd4460be504330f146d6ab 2024-11-12T07:50:08,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ce8342ee5abd4486b425666cd9f317c3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ce8342ee5abd4486b425666cd9f317c3 2024-11-12T07:50:08,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2014e53fdc38453ab770013b4933461d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2014e53fdc38453ab770013b4933461d 2024-11-12T07:50:08,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5f1d7e98b23d4ad3877ef48079ae4de0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5f1d7e98b23d4ad3877ef48079ae4de0 2024-11-12T07:50:08,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/21a32f061baf4081947893425d93db66 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/21a32f061baf4081947893425d93db66 2024-11-12T07:50:08,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/643a09363c954387a223c4820a4f063d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/643a09363c954387a223c4820a4f063d 2024-11-12T07:50:08,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6241f0ff39574455987546cebd6b7303 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/6241f0ff39574455987546cebd6b7303 2024-11-12T07:50:08,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ea7b685ceca14632950140fc9da8385f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/ea7b685ceca14632950140fc9da8385f 2024-11-12T07:50:08,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2793cdcb05564658b799db51771bc43a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/2793cdcb05564658b799db51771bc43a 2024-11-12T07:50:08,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/e45c0dc46e8e4223997547227ffefa6e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/e45c0dc46e8e4223997547227ffefa6e 2024-11-12T07:50:08,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/fb7ae337d14d4f3eae7b7ad80870115a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/fb7ae337d14d4f3eae7b7ad80870115a 2024-11-12T07:50:08,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/231c85c481ef4d67a1f41cd14cb38a41 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/231c85c481ef4d67a1f41cd14cb38a41 2024-11-12T07:50:08,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/df038f67c686407ab54646344f906746 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/df038f67c686407ab54646344f906746 2024-11-12T07:50:08,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/c23030d5cc6c46c28069e7a6dacd620a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/c23030d5cc6c46c28069e7a6dacd620a 2024-11-12T07:50:08,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/928cf3e4c6e6487497f0ab7fb4da7b34 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/928cf3e4c6e6487497f0ab7fb4da7b34 2024-11-12T07:50:08,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d77730648bed499881c48ec689edf4f3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/d77730648bed499881c48ec689edf4f3 2024-11-12T07:50:08,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f5f9cca95caa4663a5950dfb8746d326 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f5f9cca95caa4663a5950dfb8746d326 2024-11-12T07:50:08,857 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/recovered.edits/469.seqid, newMaxSeqId=469, maxSeqId=1 2024-11-12T07:50:08,857 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. 2024-11-12T07:50:08,857 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for e39e5d3040dc37fe65a005881116393f: Waiting for close lock at 1731397807524Running coprocessor pre-close hooks at 1731397807524Disabling compacts and flushes for region at 1731397807524Disabling writes for close at 1731397807524Obtaining lock to block concurrent updates at 1731397807524Preparing flush snapshotting stores in e39e5d3040dc37fe65a005881116393f at 1731397807524Finished memstore snapshotting TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731397807525 (+1 ms)Flushing stores of TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f. at 1731397807526 (+1 ms)Flushing e39e5d3040dc37fe65a005881116393f/A: creating writer at 1731397807526Flushing e39e5d3040dc37fe65a005881116393f/A: appending metadata at 1731397807530 (+4 ms)Flushing e39e5d3040dc37fe65a005881116393f/A: closing flushed file at 1731397807530Flushing e39e5d3040dc37fe65a005881116393f/B: creating writer at 1731397807949 (+419 ms)Flushing e39e5d3040dc37fe65a005881116393f/B: appending metadata at 1731397807952 (+3 ms)Flushing e39e5d3040dc37fe65a005881116393f/B: closing flushed file at 1731397807952Flushing e39e5d3040dc37fe65a005881116393f/C: creating writer at 1731397808361 (+409 ms)Flushing e39e5d3040dc37fe65a005881116393f/C: appending metadata at 1731397808363 (+2 ms)Flushing e39e5d3040dc37fe65a005881116393f/C: closing flushed file at 1731397808363Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28105525: reopening flushed file at 1731397808773 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fd0cf49: reopening flushed file at 1731397808781 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c14fd63: reopening flushed file at 1731397808788 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for e39e5d3040dc37fe65a005881116393f in 1270ms, sequenceid=466, compaction requested=true at 1731397808794 (+6 ms)Writing region close event to WAL at 1731397808854 (+60 ms)Running coprocessor post-close hooks at 1731397808857 (+3 ms)Closed at 1731397808857 2024-11-12T07:50:08,859 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed e39e5d3040dc37fe65a005881116393f 2024-11-12T07:50:08,859 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=e39e5d3040dc37fe65a005881116393f, regionState=CLOSED 2024-11-12T07:50:08,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:50:08,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-12T07:50:08,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure e39e5d3040dc37fe65a005881116393f, server=f9fc96254ef8,44299,1731397650797 in 1.4900 sec 2024-11-12T07:50:08,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-12T07:50:08,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e39e5d3040dc37fe65a005881116393f, UNASSIGN in 1.4950 sec 2024-11-12T07:50:08,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-12T07:50:08,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5000 sec 2024-11-12T07:50:08,868 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397808867"}]},"ts":"1731397808867"} 2024-11-12T07:50:08,869 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T07:50:08,869 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T07:50:08,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-12T07:50:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T07:50:09,509 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:09,511 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T07:50:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:09,515 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T07:50:09,516 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:09,519 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f 2024-11-12T07:50:09,522 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/recovered.edits] 2024-11-12T07:50:09,525 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/0594c81f39ea4e03b9e663f4bde00aec to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/0594c81f39ea4e03b9e663f4bde00aec 2024-11-12T07:50:09,526 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/65118cf11ba34c5da6cc43286fc6fe67 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/65118cf11ba34c5da6cc43286fc6fe67 2024-11-12T07:50:09,527 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/91b6ff7568ea40d8b168caf87bb15f69 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/A/91b6ff7568ea40d8b168caf87bb15f69 2024-11-12T07:50:09,528 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/09c4e6ca7098488281d4b8c0f1b59f00 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/09c4e6ca7098488281d4b8c0f1b59f00 2024-11-12T07:50:09,529 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/3182e4cf2a2b4fd092a7d884fc09a054 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/3182e4cf2a2b4fd092a7d884fc09a054 2024-11-12T07:50:09,530 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/61624f05c8c24450bb29543fbd94bc2c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/B/61624f05c8c24450bb29543fbd94bc2c 2024-11-12T07:50:09,531 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/214b1ad440424467a58979ef28f0e34d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/214b1ad440424467a58979ef28f0e34d 2024-11-12T07:50:09,532 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5d25555906f242c98fad651cad836e23 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/5d25555906f242c98fad651cad836e23 2024-11-12T07:50:09,532 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f4ed906a27a140508624777996ee0343 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/C/f4ed906a27a140508624777996ee0343 2024-11-12T07:50:09,534 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/recovered.edits/469.seqid to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f/recovered.edits/469.seqid 2024-11-12T07:50:09,534 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/e39e5d3040dc37fe65a005881116393f 2024-11-12T07:50:09,534 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T07:50:09,536 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:09,538 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T07:50:09,540 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T07:50:09,541 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:09,541 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T07:50:09,541 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731397809541"}]},"ts":"9223372036854775807"} 2024-11-12T07:50:09,543 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T07:50:09,543 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e39e5d3040dc37fe65a005881116393f, NAME => 'TestAcidGuarantees,,1731397776637.e39e5d3040dc37fe65a005881116393f.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T07:50:09,543 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T07:50:09,544 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731397809543"}]},"ts":"9223372036854775807"} 2024-11-12T07:50:09,545 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T07:50:09,546 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:09,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-11-12T07:50:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T07:50:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T07:50:09,629 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:09,644 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 248), OpenFileDescriptor=441 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=289 (was 338), ProcessCount=11 (was 11), AvailableMemoryMB=5784 (was 5822) 2024-11-12T07:50:09,655 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=289, ProcessCount=11, AvailableMemoryMB=5784 2024-11-12T07:50:09,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:50:09,656 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:50:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:09,659 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T07:50:09,659 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:09,659 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-12T07:50:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T07:50:09,660 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T07:50:09,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742311_1487 (size=1061) 2024-11-12T07:50:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T07:50:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T07:50:10,070 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f 2024-11-12T07:50:10,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742312_1488 (size=53) 2024-11-12T07:50:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T07:50:10,375 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T07:50:10,478 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:10,478 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1c3bb6ad593aa45887e06136ab72eae4, disabling compactions & flushes 2024-11-12T07:50:10,478 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:10,478 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:10,478 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. after waiting 0 ms 2024-11-12T07:50:10,478 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:10,478 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:10,478 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1c3bb6ad593aa45887e06136ab72eae4: Waiting for close lock at 1731397810478Disabling compacts and flushes for region at 1731397810478Disabling writes for close at 1731397810478Writing region close event to WAL at 1731397810478Closed at 1731397810478 2024-11-12T07:50:10,480 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T07:50:10,480 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731397810480"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731397810480"}]},"ts":"1731397810480"} 2024-11-12T07:50:10,483 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T07:50:10,484 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T07:50:10,485 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397810484"}]},"ts":"1731397810484"} 2024-11-12T07:50:10,487 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T07:50:10,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, ASSIGN}] 2024-11-12T07:50:10,488 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, ASSIGN 2024-11-12T07:50:10,489 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, ASSIGN; state=OFFLINE, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=false 2024-11-12T07:50:10,640 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1c3bb6ad593aa45887e06136ab72eae4, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:10,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, ASSIGN because future has completed 2024-11-12T07:50:10,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:50:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T07:50:10,802 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:10,802 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:10,802 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,802 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:10,803 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,803 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,805 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,807 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:50:10,807 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c3bb6ad593aa45887e06136ab72eae4 columnFamilyName A 2024-11-12T07:50:10,807 DEBUG [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:10,808 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(327): Store=1c3bb6ad593aa45887e06136ab72eae4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:10,808 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,810 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:50:10,810 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c3bb6ad593aa45887e06136ab72eae4 columnFamilyName B 2024-11-12T07:50:10,810 DEBUG [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:10,811 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(327): Store=1c3bb6ad593aa45887e06136ab72eae4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:10,811 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,813 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:50:10,813 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c3bb6ad593aa45887e06136ab72eae4 columnFamilyName C 2024-11-12T07:50:10,813 DEBUG [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:10,814 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(327): Store=1c3bb6ad593aa45887e06136ab72eae4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:10,814 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:10,814 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,815 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,816 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,818 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,818 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,819 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:50:10,820 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,823 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T07:50:10,824 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 1c3bb6ad593aa45887e06136ab72eae4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59101926, jitterRate=-0.11931267380714417}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:50:10,824 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:10,824 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 1c3bb6ad593aa45887e06136ab72eae4: Running coprocessor pre-open hook at 1731397810803Writing region info on filesystem at 1731397810803Initializing all the Stores at 1731397810804 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397810804Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397810804Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397810805 (+1 ms)Cleaning up temporary data from old regions at 1731397810818 (+13 ms)Running coprocessor post-open hooks at 1731397810824 (+6 ms)Region opened successfully at 1731397810824 2024-11-12T07:50:10,825 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., pid=138, masterSystemTime=1731397810798 2024-11-12T07:50:10,828 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:10,828 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:10,829 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=1c3bb6ad593aa45887e06136ab72eae4, regionState=OPEN, openSeqNum=2, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:10,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:50:10,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-12T07:50:10,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 in 188 msec 2024-11-12T07:50:10,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-12T07:50:10,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, ASSIGN in 348 msec 2024-11-12T07:50:10,838 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T07:50:10,839 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397810838"}]},"ts":"1731397810838"} 2024-11-12T07:50:10,840 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T07:50:10,841 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T07:50:10,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-12T07:50:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T07:50:11,808 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:11,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:11,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:11,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:11,810 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775708ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:11,812 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bad34c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:11,812 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:11,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:11,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:11,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:11,814 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T07:50:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is f9fc96254ef8,43319,1731397650023 2024-11-12T07:50:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5aee35ef 2024-11-12T07:50:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T07:50:11,816 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T07:50:11,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T07:50:11,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T07:50:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:11,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742313_1489 (size=1097) 2024-11-12T07:50:12,227 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T07:50:12,227 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T07:50:12,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:50:12,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, REOPEN/MOVE}] 2024-11-12T07:50:12,231 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, REOPEN/MOVE 2024-11-12T07:50:12,232 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1c3bb6ad593aa45887e06136ab72eae4, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, REOPEN/MOVE because future has completed 2024-11-12T07:50:12,233 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:50:12,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:50:12,386 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,386 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:50:12,387 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 1c3bb6ad593aa45887e06136ab72eae4, disabling compactions & flushes 2024-11-12T07:50:12,387 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,387 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,387 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. after waiting 0 ms 2024-11-12T07:50:12,387 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,394 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T07:50:12,395 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,395 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 1c3bb6ad593aa45887e06136ab72eae4: Waiting for close lock at 1731397812386Running coprocessor pre-close hooks at 1731397812386Disabling compacts and flushes for region at 1731397812387 (+1 ms)Disabling writes for close at 1731397812387Writing region close event to WAL at 1731397812388 (+1 ms)Running coprocessor post-close hooks at 1731397812395 (+7 ms)Closed at 1731397812395 2024-11-12T07:50:12,395 WARN [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 1c3bb6ad593aa45887e06136ab72eae4 to self. 2024-11-12T07:50:12,398 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1c3bb6ad593aa45887e06136ab72eae4, regionState=CLOSED 2024-11-12T07:50:12,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:50:12,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-12T07:50:12,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 in 170 msec 2024-11-12T07:50:12,406 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, REOPEN/MOVE; state=CLOSED, location=f9fc96254ef8,44299,1731397650797; forceNewPlan=false, retain=true 2024-11-12T07:50:12,557 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1c3bb6ad593aa45887e06136ab72eae4, regionState=OPENING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, REOPEN/MOVE because future has completed 2024-11-12T07:50:12,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:50:12,719 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,719 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} 2024-11-12T07:50:12,719 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,719 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T07:50:12,720 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,720 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,723 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,724 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:50:12,724 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c3bb6ad593aa45887e06136ab72eae4 columnFamilyName A 2024-11-12T07:50:12,726 DEBUG [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:12,727 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(327): Store=1c3bb6ad593aa45887e06136ab72eae4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:12,727 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,727 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:50:12,728 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c3bb6ad593aa45887e06136ab72eae4 columnFamilyName B 2024-11-12T07:50:12,728 DEBUG [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:12,728 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(327): Store=1c3bb6ad593aa45887e06136ab72eae4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:12,728 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,729 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T07:50:12,729 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c3bb6ad593aa45887e06136ab72eae4 columnFamilyName C 2024-11-12T07:50:12,729 DEBUG [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:12,729 INFO [StoreOpener-1c3bb6ad593aa45887e06136ab72eae4-1 {}] regionserver.HStore(327): Store=1c3bb6ad593aa45887e06136ab72eae4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T07:50:12,729 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,729 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,730 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,731 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,732 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,732 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,733 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T07:50:12,734 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,735 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 1c3bb6ad593aa45887e06136ab72eae4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59718795, jitterRate=-0.11012060940265656}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T07:50:12,735 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,736 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 1c3bb6ad593aa45887e06136ab72eae4: Running coprocessor pre-open hook at 1731397812720Writing region info on filesystem at 1731397812720Initializing all the Stores at 1731397812721 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397812721Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397812722 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731397812722Cleaning up temporary data from old regions at 1731397812732 (+10 ms)Running coprocessor post-open hooks at 1731397812735 (+3 ms)Region opened successfully at 1731397812736 (+1 ms) 2024-11-12T07:50:12,737 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., pid=143, masterSystemTime=1731397812714 2024-11-12T07:50:12,738 DEBUG [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,738 INFO [RS_OPEN_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,739 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=1c3bb6ad593aa45887e06136ab72eae4, regionState=OPEN, openSeqNum=5, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:50:12,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-12T07:50:12,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 in 180 msec 2024-11-12T07:50:12,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-12T07:50:12,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, REOPEN/MOVE in 511 msec 2024-11-12T07:50:12,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-12T07:50:12,746 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-11-12T07:50:12,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-11-12T07:50:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T07:50:12,748 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:12,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bcb50d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,750 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7d0ccd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392132f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,753 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd6a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,755 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c96bf26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45067ad2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,758 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9da4c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b044dea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c769670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57a045ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,764 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f6e519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20415c60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ade1a04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed58018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,770 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77089a33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,771 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dfd4800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7783151b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e2f9e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f9fc96254ef8,43319,-1 for getting cluster id 2024-11-12T07:50:12,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T07:50:12,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '402938bb-bb5a-4f15-b60d-61d4ad603f05' 2024-11-12T07:50:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T07:50:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "402938bb-bb5a-4f15-b60d-61d4ad603f05" 2024-11-12T07:50:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d34c99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f9fc96254ef8,43319,-1] 2024-11-12T07:50:12,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T07:50:12,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:12,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T07:50:12,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d6d8d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T07:50:12,777 DEBUG [Thread-2224 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,777 DEBUG [Thread-2220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,777 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:50:12,778 DEBUG [Thread-2218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,778 DEBUG [Thread-2209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,778 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-12T07:50:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,779 DEBUG [Thread-2213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,779 DEBUG [Thread-2211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T07:50:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,780 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:50:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,780 DEBUG [Thread-2205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,781 DEBUG [Thread-2207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,781 DEBUG [Thread-2222 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T07:50:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,781 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:50:12,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:50:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f9fc96254ef8,44299,1731397650797, seqNum=-1] 2024-11-12T07:50:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T07:50:12,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T07:50:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5] 2024-11-12T07:50:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:50:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:12,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397812808, max timestamp=1731397812822Num uniques 859; 2024-11-12T07:50:12,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:50:12,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731397812808, max timestamp=1731397812822Num uniques 859; 2024-11-12T07:50:12,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T07:50:12,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397812808, max timestamp=1731397812822Num uniques 859; 2024-11-12T07:50:12,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:50:12,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125e2c23688fed4e5c9216e271bf97788e_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397812796/Put/seqid=0 2024-11-12T07:50:12,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742314_1490 (size=12154) 2024-11-12T07:50:12,837 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:12,840 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125e2c23688fed4e5c9216e271bf97788e_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125e2c23688fed4e5c9216e271bf97788e_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:12,841 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/a6f64658afa846e38defa57b9b9eb033, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:12,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/a6f64658afa846e38defa57b9b9eb033 is 175, key is test_row_0/A:col10/1731397812796/Put/seqid=0 2024-11-12T07:50:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397872837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397872838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397872839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397872840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397872841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742315_1491 (size=30955) 2024-11-12T07:50:12,857 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/a6f64658afa846e38defa57b9b9eb033 2024-11-12T07:50:12,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/846de35f5c9e4a398fe2053bc8bc8c1d is 50, key is test_row_0/B:col10/1731397812796/Put/seqid=0 2024-11-12T07:50:12,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T07:50:12,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742316_1492 (size=12001) 2024-11-12T07:50:12,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T07:50:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:12,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:12,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:12,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397872948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397872948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397872948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397872948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397872957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T07:50:13,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:13,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T07:50:13,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397873158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397873158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397873158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397873158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397873167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T07:50:13,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:13,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/846de35f5c9e4a398fe2053bc8bc8c1d 2024-11-12T07:50:13,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/ae447f8cd972402996186d904298daa2 is 50, key is test_row_0/C:col10/1731397812796/Put/seqid=0 2024-11-12T07:50:13,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742317_1493 (size=12001) 2024-11-12T07:50:13,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T07:50:13,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:13,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T07:50:13,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397873468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397873468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397873468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397873468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397873478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T07:50:13,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:13,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T07:50:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:13,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/ae447f8cd972402996186d904298daa2 2024-11-12T07:50:13,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/a6f64658afa846e38defa57b9b9eb033 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/a6f64658afa846e38defa57b9b9eb033 2024-11-12T07:50:13,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/a6f64658afa846e38defa57b9b9eb033, entries=150, sequenceid=15, filesize=30.2 K 2024-11-12T07:50:13,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/846de35f5c9e4a398fe2053bc8bc8c1d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/846de35f5c9e4a398fe2053bc8bc8c1d 2024-11-12T07:50:13,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/846de35f5c9e4a398fe2053bc8bc8c1d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-12T07:50:13,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/ae447f8cd972402996186d904298daa2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ae447f8cd972402996186d904298daa2 2024-11-12T07:50:13,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ae447f8cd972402996186d904298daa2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-12T07:50:13,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 1c3bb6ad593aa45887e06136ab72eae4 in 956ms, sequenceid=15, compaction requested=false 2024-11-12T07:50:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T07:50:13,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:13,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T07:50:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:13,861 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-12T07:50:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:13,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:13,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111287cbf844ea734e9d8f7f3a123e889370_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397812838/Put/seqid=0 2024-11-12T07:50:13,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742318_1494 (size=12154) 2024-11-12T07:50:13,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:13,881 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111287cbf844ea734e9d8f7f3a123e889370_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111287cbf844ea734e9d8f7f3a123e889370_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:13,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/20dfe610905746adbab93c73cdcc4206, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:13,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/20dfe610905746adbab93c73cdcc4206 is 175, key is test_row_0/A:col10/1731397812838/Put/seqid=0 2024-11-12T07:50:13,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742319_1495 (size=30955) 2024-11-12T07:50:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T07:50:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:13,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:13,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397873986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397873987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397873988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,992 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397873988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:13,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397873990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397874098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397874098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397874098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397874098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,296 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/20dfe610905746adbab93c73cdcc4206 2024-11-12T07:50:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/9d7540d6884b4f6e88a6a4a96c20989b is 50, key is test_row_0/B:col10/1731397812838/Put/seqid=0 2024-11-12T07:50:14,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397874308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397874308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397874308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397874308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742320_1496 (size=12001) 2024-11-12T07:50:14,380 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T07:50:14,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397874617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397874618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397874618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397874618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:14,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:14,718 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/9d7540d6884b4f6e88a6a4a96c20989b 2024-11-12T07:50:14,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/088c290d1a51409a97e8be48270d21b3 is 50, key is test_row_0/C:col10/1731397812838/Put/seqid=0 2024-11-12T07:50:14,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742321_1497 (size=12001) 2024-11-12T07:50:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T07:50:15,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397875008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:15,010 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 1c3bb6ad593aa45887e06136ab72eae4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:50:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:15,129 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/088c290d1a51409a97e8be48270d21b3 2024-11-12T07:50:15,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397875128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397875128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:15,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397875128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397875128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/20dfe610905746adbab93c73cdcc4206 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/20dfe610905746adbab93c73cdcc4206 2024-11-12T07:50:15,136 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/20dfe610905746adbab93c73cdcc4206, entries=150, sequenceid=50, filesize=30.2 K 2024-11-12T07:50:15,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/9d7540d6884b4f6e88a6a4a96c20989b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d7540d6884b4f6e88a6a4a96c20989b 2024-11-12T07:50:15,140 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d7540d6884b4f6e88a6a4a96c20989b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T07:50:15,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/088c290d1a51409a97e8be48270d21b3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/088c290d1a51409a97e8be48270d21b3 2024-11-12T07:50:15,145 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/088c290d1a51409a97e8be48270d21b3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-12T07:50:15,145 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=53.67 KB/54960 for 1c3bb6ad593aa45887e06136ab72eae4 in 1284ms, sequenceid=50, compaction requested=false 2024-11-12T07:50:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-12T07:50:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-12T07:50:15,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-12T07:50:15,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3650 sec 2024-11-12T07:50:15,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.3730 sec 2024-11-12T07:50:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:16,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:50:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:16,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:16,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122c801c420ba647628eb264d78642736e_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397816139/Put/seqid=0 2024-11-12T07:50:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742322_1498 (size=14594) 2024-11-12T07:50:16,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397876160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397876161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397876162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397876162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397876267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397876268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397876268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397876268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397876477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397876478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397876478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397876478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,550 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:16,553 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122c801c420ba647628eb264d78642736e_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c801c420ba647628eb264d78642736e_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:16,554 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/d0e98e92abcd43589ef7e4aa6f4e1421, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:16,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/d0e98e92abcd43589ef7e4aa6f4e1421 is 175, key is test_row_0/A:col10/1731397816139/Put/seqid=0 2024-11-12T07:50:16,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742323_1499 (size=39549) 2024-11-12T07:50:16,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397876788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397876788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397876788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397876789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:16,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T07:50:16,948 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:16,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:50:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-12T07:50:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T07:50:16,953 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:50:16,954 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:50:16,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:50:16,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/d0e98e92abcd43589ef7e4aa6f4e1421 2024-11-12T07:50:16,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e47509678e194824bacceb7fc72bdfd6 is 50, key is test_row_0/B:col10/1731397816139/Put/seqid=0 2024-11-12T07:50:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742324_1500 (size=12001) 2024-11-12T07:50:17,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397877017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:17,020 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 1c3bb6ad593aa45887e06136ab72eae4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:50:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T07:50:17,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T07:50:17,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:17,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T07:50:17,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T07:50:17,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:17,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397877297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397877298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397877298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:17,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397877298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:17,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e47509678e194824bacceb7fc72bdfd6 2024-11-12T07:50:17,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/77115b55f66a414b8c0ec1560075f395 is 50, key is test_row_0/C:col10/1731397816139/Put/seqid=0 2024-11-12T07:50:17,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742325_1501 (size=12001) 2024-11-12T07:50:17,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T07:50:17,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:17,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T07:50:17,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T07:50:17,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T07:50:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:17,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/77115b55f66a414b8c0ec1560075f395 2024-11-12T07:50:17,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/d0e98e92abcd43589ef7e4aa6f4e1421 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/d0e98e92abcd43589ef7e4aa6f4e1421 2024-11-12T07:50:17,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/d0e98e92abcd43589ef7e4aa6f4e1421, entries=200, sequenceid=63, filesize=38.6 K 2024-11-12T07:50:17,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e47509678e194824bacceb7fc72bdfd6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e47509678e194824bacceb7fc72bdfd6 2024-11-12T07:50:17,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e47509678e194824bacceb7fc72bdfd6, entries=150, sequenceid=63, filesize=11.7 K 2024-11-12T07:50:17,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/77115b55f66a414b8c0ec1560075f395 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/77115b55f66a414b8c0ec1560075f395 2024-11-12T07:50:17,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/77115b55f66a414b8c0ec1560075f395, entries=150, sequenceid=63, filesize=11.7 K 2024-11-12T07:50:17,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1c3bb6ad593aa45887e06136ab72eae4 in 1711ms, sequenceid=63, compaction requested=true 2024-11-12T07:50:17,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:17,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:50:17,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:17,852 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:17,852 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:17,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:50:17,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:17,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:50:17,853 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:17,853 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/A is initiating minor compaction (all files) 2024-11-12T07:50:17,853 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/A in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,853 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/a6f64658afa846e38defa57b9b9eb033, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/20dfe610905746adbab93c73cdcc4206, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/d0e98e92abcd43589ef7e4aa6f4e1421] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=99.1 K 2024-11-12T07:50:17,853 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,853 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/a6f64658afa846e38defa57b9b9eb033, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/20dfe610905746adbab93c73cdcc4206, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/d0e98e92abcd43589ef7e4aa6f4e1421] 2024-11-12T07:50:17,854 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:17,854 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/B is initiating minor compaction (all files) 2024-11-12T07:50:17,854 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/B in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,854 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/846de35f5c9e4a398fe2053bc8bc8c1d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d7540d6884b4f6e88a6a4a96c20989b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e47509678e194824bacceb7fc72bdfd6] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=35.2 K 2024-11-12T07:50:17,854 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f64658afa846e38defa57b9b9eb033, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731397812795 2024-11-12T07:50:17,854 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 846de35f5c9e4a398fe2053bc8bc8c1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731397812795 2024-11-12T07:50:17,854 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20dfe610905746adbab93c73cdcc4206, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731397812836 2024-11-12T07:50:17,855 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d7540d6884b4f6e88a6a4a96c20989b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731397812836 2024-11-12T07:50:17,855 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e98e92abcd43589ef7e4aa6f4e1421, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731397813979 2024-11-12T07:50:17,855 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e47509678e194824bacceb7fc72bdfd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731397813988 2024-11-12T07:50:17,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:17,875 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:17,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T07:50:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,886 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T07:50:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:17,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:17,893 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:17,893 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/16f6b61562404c7fbc1a5057b8905775 is 50, key is test_row_0/B:col10/1731397816139/Put/seqid=0 2024-11-12T07:50:17,911 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112d2e1afcd40d949bd93d214ba9932a32b_1c3bb6ad593aa45887e06136ab72eae4 store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:17,913 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112d2e1afcd40d949bd93d214ba9932a32b_1c3bb6ad593aa45887e06136ab72eae4, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:17,914 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d2e1afcd40d949bd93d214ba9932a32b_1c3bb6ad593aa45887e06136ab72eae4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:17,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e295eaa2afd343b380399dd676ac35b3_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397816160/Put/seqid=0 2024-11-12T07:50:17,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742326_1502 (size=12104) 2024-11-12T07:50:17,984 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/16f6b61562404c7fbc1a5057b8905775 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/16f6b61562404c7fbc1a5057b8905775 2024-11-12T07:50:17,992 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/B of 1c3bb6ad593aa45887e06136ab72eae4 into 16f6b61562404c7fbc1a5057b8905775(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:17,992 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:17,992 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/B, priority=13, startTime=1731397817852; duration=0sec 2024-11-12T07:50:17,992 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:17,993 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:B 2024-11-12T07:50:17,993 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:17,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742327_1503 (size=4469) 2024-11-12T07:50:17,994 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:17,994 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/C is initiating minor compaction (all files) 2024-11-12T07:50:17,994 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/C in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:17,994 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ae447f8cd972402996186d904298daa2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/088c290d1a51409a97e8be48270d21b3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/77115b55f66a414b8c0ec1560075f395] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=35.2 K 2024-11-12T07:50:17,994 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ae447f8cd972402996186d904298daa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731397812795 2024-11-12T07:50:17,995 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 088c290d1a51409a97e8be48270d21b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731397812836 2024-11-12T07:50:17,995 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 77115b55f66a414b8c0ec1560075f395, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731397813988 2024-11-12T07:50:18,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742328_1504 (size=12154) 2024-11-12T07:50:18,025 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:18,025 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/3502587c35ac47e1b1d1a5cd718ba3b9 is 50, key is test_row_0/C:col10/1731397816139/Put/seqid=0 2024-11-12T07:50:18,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742329_1505 (size=12104) 2024-11-12T07:50:18,085 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/3502587c35ac47e1b1d1a5cd718ba3b9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3502587c35ac47e1b1d1a5cd718ba3b9 2024-11-12T07:50:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T07:50:18,091 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/C of 1c3bb6ad593aa45887e06136ab72eae4 into 3502587c35ac47e1b1d1a5cd718ba3b9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:18,091 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:18,091 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/C, priority=13, startTime=1731397817852; duration=0sec 2024-11-12T07:50:18,091 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:18,091 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:C 2024-11-12T07:50:18,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:18,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397878327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397878330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397878332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397878333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,395 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#A#compaction#426 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:18,396 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/1cf4448acbb4468f94b34b5b18309656 is 175, key is test_row_0/A:col10/1731397816139/Put/seqid=0 2024-11-12T07:50:18,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:18,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742330_1506 (size=31058) 2024-11-12T07:50:18,434 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e295eaa2afd343b380399dd676ac35b3_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e295eaa2afd343b380399dd676ac35b3_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:18,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/30761099303148679a494af9dd5b5f44, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/30761099303148679a494af9dd5b5f44 is 175, key is test_row_0/A:col10/1731397816160/Put/seqid=0 2024-11-12T07:50:18,436 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/1cf4448acbb4468f94b34b5b18309656 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/1cf4448acbb4468f94b34b5b18309656 2024-11-12T07:50:18,440 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/A of 1c3bb6ad593aa45887e06136ab72eae4 into 1cf4448acbb4468f94b34b5b18309656(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:18,440 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:18,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,440 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/A, priority=13, startTime=1731397817852; duration=0sec 2024-11-12T07:50:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397878438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,440 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:18,440 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:A 2024-11-12T07:50:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397878438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742331_1507 (size=30955) 2024-11-12T07:50:18,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397878447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397878447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397878648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397878648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397878658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397878658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,845 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/30761099303148679a494af9dd5b5f44 2024-11-12T07:50:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/cdf18663793c4cd3a3897db9def49cd0 is 50, key is test_row_0/B:col10/1731397816160/Put/seqid=0 2024-11-12T07:50:18,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742332_1508 (size=12001) 2024-11-12T07:50:18,900 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/cdf18663793c4cd3a3897db9def49cd0 2024-11-12T07:50:18,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/f28dcfb7eaf84ddabe64d511da9c0716 is 50, key is test_row_0/C:col10/1731397816160/Put/seqid=0 2024-11-12T07:50:18,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742333_1509 (size=12001) 2024-11-12T07:50:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397878959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397878959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397878968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397878968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T07:50:19,348 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/f28dcfb7eaf84ddabe64d511da9c0716 2024-11-12T07:50:19,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/30761099303148679a494af9dd5b5f44 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/30761099303148679a494af9dd5b5f44 2024-11-12T07:50:19,354 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/30761099303148679a494af9dd5b5f44, entries=150, sequenceid=82, filesize=30.2 K 2024-11-12T07:50:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/cdf18663793c4cd3a3897db9def49cd0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/cdf18663793c4cd3a3897db9def49cd0 2024-11-12T07:50:19,358 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/cdf18663793c4cd3a3897db9def49cd0, entries=150, sequenceid=82, filesize=11.7 K 2024-11-12T07:50:19,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/f28dcfb7eaf84ddabe64d511da9c0716 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/f28dcfb7eaf84ddabe64d511da9c0716 2024-11-12T07:50:19,361 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/f28dcfb7eaf84ddabe64d511da9c0716, entries=150, sequenceid=82, filesize=11.7 K 2024-11-12T07:50:19,362 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1c3bb6ad593aa45887e06136ab72eae4 in 1476ms, sequenceid=82, compaction requested=false 2024-11-12T07:50:19,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:19,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:19,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-12T07:50:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-12T07:50:19,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-12T07:50:19,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4090 sec 2024-11-12T07:50:19,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.4160 sec 2024-11-12T07:50:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:19,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T07:50:19,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:19,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:19,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:19,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:19,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:19,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:19,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129cef468039e74424b6567c134a9c48bc_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397819469/Put/seqid=0 2024-11-12T07:50:19,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742334_1510 (size=14594) 2024-11-12T07:50:19,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397879500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397879500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397879503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397879505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397879608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397879608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397879618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397879618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397879818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397879818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397879828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397879828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:19,882 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:19,886 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129cef468039e74424b6567c134a9c48bc_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129cef468039e74424b6567c134a9c48bc_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:19,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/4c6f68dc31f2484bb64dbe84f761164f, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:19,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/4c6f68dc31f2484bb64dbe84f761164f is 175, key is test_row_0/A:col10/1731397819469/Put/seqid=0 2024-11-12T07:50:19,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742335_1511 (size=39549) 2024-11-12T07:50:19,896 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/4c6f68dc31f2484bb64dbe84f761164f 2024-11-12T07:50:19,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/9d5e8be2d8294fddb63c3d65891c8f18 is 50, key is test_row_0/B:col10/1731397819469/Put/seqid=0 2024-11-12T07:50:19,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742336_1512 (size=12001) 2024-11-12T07:50:19,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/9d5e8be2d8294fddb63c3d65891c8f18 2024-11-12T07:50:19,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/9cdc3e9a00af46d1a539535ec1558486 is 50, key is test_row_0/C:col10/1731397819469/Put/seqid=0 2024-11-12T07:50:19,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742337_1513 (size=12001) 2024-11-12T07:50:19,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/9cdc3e9a00af46d1a539535ec1558486 2024-11-12T07:50:20,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/4c6f68dc31f2484bb64dbe84f761164f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4c6f68dc31f2484bb64dbe84f761164f 2024-11-12T07:50:20,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4c6f68dc31f2484bb64dbe84f761164f, entries=200, sequenceid=100, filesize=38.6 K 2024-11-12T07:50:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/9d5e8be2d8294fddb63c3d65891c8f18 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d5e8be2d8294fddb63c3d65891c8f18 2024-11-12T07:50:20,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d5e8be2d8294fddb63c3d65891c8f18, entries=150, sequenceid=100, filesize=11.7 K 2024-11-12T07:50:20,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/9cdc3e9a00af46d1a539535ec1558486 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9cdc3e9a00af46d1a539535ec1558486 2024-11-12T07:50:20,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9cdc3e9a00af46d1a539535ec1558486, entries=150, sequenceid=100, filesize=11.7 K 2024-11-12T07:50:20,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1c3bb6ad593aa45887e06136ab72eae4 in 546ms, sequenceid=100, compaction requested=true 2024-11-12T07:50:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:50:20,017 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:50:20,017 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:50:20,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:20,018 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:20,018 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/B is initiating minor compaction (all files) 2024-11-12T07:50:20,018 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/B in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:20,018 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/16f6b61562404c7fbc1a5057b8905775, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/cdf18663793c4cd3a3897db9def49cd0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d5e8be2d8294fddb63c3d65891c8f18] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=35.3 K 2024-11-12T07:50:20,018 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:20,018 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/A is initiating minor compaction (all files) 2024-11-12T07:50:20,018 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/A in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:20,018 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/1cf4448acbb4468f94b34b5b18309656, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/30761099303148679a494af9dd5b5f44, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4c6f68dc31f2484bb64dbe84f761164f] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=99.2 K 2024-11-12T07:50:20,018 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f6b61562404c7fbc1a5057b8905775, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731397813988 2024-11-12T07:50:20,018 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:20,019 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/1cf4448acbb4468f94b34b5b18309656, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/30761099303148679a494af9dd5b5f44, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4c6f68dc31f2484bb64dbe84f761164f] 2024-11-12T07:50:20,019 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf18663793c4cd3a3897db9def49cd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731397816159 2024-11-12T07:50:20,020 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf4448acbb4468f94b34b5b18309656, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731397813988 2024-11-12T07:50:20,020 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d5e8be2d8294fddb63c3d65891c8f18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731397818330 2024-11-12T07:50:20,020 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30761099303148679a494af9dd5b5f44, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731397816159 2024-11-12T07:50:20,020 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c6f68dc31f2484bb64dbe84f761164f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731397818330 2024-11-12T07:50:20,028 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:20,028 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/7a4f71d4bc0b4265a3d63e47b0a3c7c8 is 50, key is test_row_0/B:col10/1731397819469/Put/seqid=0 2024-11-12T07:50:20,029 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:20,031 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112aaaec898e3114fa6b0c6372d830bcd10_1c3bb6ad593aa45887e06136ab72eae4 store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:20,033 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112aaaec898e3114fa6b0c6372d830bcd10_1c3bb6ad593aa45887e06136ab72eae4, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:20,034 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112aaaec898e3114fa6b0c6372d830bcd10_1c3bb6ad593aa45887e06136ab72eae4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:20,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742339_1515 (size=4469) 2024-11-12T07:50:20,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742338_1514 (size=12207) 2024-11-12T07:50:20,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T07:50:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:20,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:20,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:20,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:20,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:20,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:20,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:20,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127e86ebd6576e4b7d95ccf43a0eb3637f_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397820130/Put/seqid=0 2024-11-12T07:50:20,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742340_1516 (size=17034) 2024-11-12T07:50:20,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:20,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397880153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397880152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397880154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397880155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127e86ebd6576e4b7d95ccf43a0eb3637f_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e86ebd6576e4b7d95ccf43a0eb3637f_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:20,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/4644933fd1ad40f98247a01c8253c365, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:20,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/4644933fd1ad40f98247a01c8253c365 is 175, key is test_row_0/A:col10/1731397820130/Put/seqid=0 2024-11-12T07:50:20,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742341_1517 (size=48139) 2024-11-12T07:50:20,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397880267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397880268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397880268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397880268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,375 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T07:50:20,375 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T07:50:20,446 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#A#compaction#436 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:20,446 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e49fe669105c42fea7c0322900a6c362 is 175, key is test_row_0/A:col10/1731397819469/Put/seqid=0 2024-11-12T07:50:20,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742342_1518 (size=31161) 2024-11-12T07:50:20,465 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/7a4f71d4bc0b4265a3d63e47b0a3c7c8 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/7a4f71d4bc0b4265a3d63e47b0a3c7c8 2024-11-12T07:50:20,468 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/B of 1c3bb6ad593aa45887e06136ab72eae4 into 7a4f71d4bc0b4265a3d63e47b0a3c7c8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:20,468 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:20,468 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/B, priority=13, startTime=1731397820017; duration=0sec 2024-11-12T07:50:20,468 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:20,468 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:B 2024-11-12T07:50:20,468 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:20,469 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:20,469 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/C is initiating minor compaction (all files) 2024-11-12T07:50:20,469 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/C in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:20,469 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3502587c35ac47e1b1d1a5cd718ba3b9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/f28dcfb7eaf84ddabe64d511da9c0716, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9cdc3e9a00af46d1a539535ec1558486] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=35.3 K 2024-11-12T07:50:20,469 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 3502587c35ac47e1b1d1a5cd718ba3b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731397813988 2024-11-12T07:50:20,470 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f28dcfb7eaf84ddabe64d511da9c0716, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731397816159 2024-11-12T07:50:20,470 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cdc3e9a00af46d1a539535ec1558486, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731397818330 2024-11-12T07:50:20,474 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:20,475 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/489f2083d899437f9aa83ddcf7446645 is 50, key is test_row_0/C:col10/1731397819469/Put/seqid=0 2024-11-12T07:50:20,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742343_1519 (size=12207) 2024-11-12T07:50:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397880477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397880478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,482 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/489f2083d899437f9aa83ddcf7446645 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/489f2083d899437f9aa83ddcf7446645 2024-11-12T07:50:20,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397880478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397880478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,487 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/C of 1c3bb6ad593aa45887e06136ab72eae4 into 489f2083d899437f9aa83ddcf7446645(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:20,487 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:20,487 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/C, priority=13, startTime=1731397820017; duration=0sec 2024-11-12T07:50:20,487 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:20,487 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:C 2024-11-12T07:50:20,576 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/4644933fd1ad40f98247a01c8253c365 2024-11-12T07:50:20,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e760044bb0c94c7aaa1d91ead9a87c23 is 50, key is test_row_0/B:col10/1731397820130/Put/seqid=0 2024-11-12T07:50:20,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742344_1520 (size=12001) 2024-11-12T07:50:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397880788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397880789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397880789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397880789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:20,854 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e49fe669105c42fea7c0322900a6c362 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e49fe669105c42fea7c0322900a6c362 2024-11-12T07:50:20,858 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/A of 1c3bb6ad593aa45887e06136ab72eae4 into e49fe669105c42fea7c0322900a6c362(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:20,858 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:20,858 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/A, priority=13, startTime=1731397820017; duration=0sec 2024-11-12T07:50:20,858 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:20,858 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:A 2024-11-12T07:50:20,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e760044bb0c94c7aaa1d91ead9a87c23 2024-11-12T07:50:20,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe is 50, key is test_row_0/C:col10/1731397820130/Put/seqid=0 2024-11-12T07:50:20,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742345_1521 (size=12001) 2024-11-12T07:50:21,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397881057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:21,058 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_1' in 1c3bb6ad593aa45887e06136ab72eae4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:50:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T07:50:21,108 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:21,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:50:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-12T07:50:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T07:50:21,111 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:50:21,111 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:50:21,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:50:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T07:50:21,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T07:50:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:21,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:21,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397881298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:21,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397881298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397881298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397881298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:21,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:21,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe 2024-11-12T07:50:21,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/4644933fd1ad40f98247a01c8253c365 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4644933fd1ad40f98247a01c8253c365 2024-11-12T07:50:21,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4644933fd1ad40f98247a01c8253c365, entries=250, sequenceid=119, filesize=47.0 K 2024-11-12T07:50:21,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e760044bb0c94c7aaa1d91ead9a87c23 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e760044bb0c94c7aaa1d91ead9a87c23 2024-11-12T07:50:21,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e760044bb0c94c7aaa1d91ead9a87c23, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T07:50:21,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe 2024-11-12T07:50:21,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T07:50:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:21,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:21,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:21,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T07:50:21,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1c3bb6ad593aa45887e06136ab72eae4 in 1290ms, sequenceid=119, compaction requested=false 2024-11-12T07:50:21,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T07:50:21,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T07:50:21,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:21,573 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T07:50:21,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:21,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:21,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:21,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:21,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:21,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:21,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122dd53deaf1474b57942384f6fc7f7096_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397820153/Put/seqid=0 2024-11-12T07:50:21,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742346_1522 (size=12254) 2024-11-12T07:50:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,605 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122dd53deaf1474b57942384f6fc7f7096_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122dd53deaf1474b57942384f6fc7f7096_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:21,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/6604837e6f2e49dbbc87044e3ace4c76, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/6604837e6f2e49dbbc87044e3ace4c76 is 175, key is test_row_0/A:col10/1731397820153/Put/seqid=0 2024-11-12T07:50:21,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742347_1523 (size=31055) 2024-11-12T07:50:21,617 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/6604837e6f2e49dbbc87044e3ace4c76 2024-11-12T07:50:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/d170d352c3924632987aa439e7c3c3b4 is 50, key is test_row_0/B:col10/1731397820153/Put/seqid=0 2024-11-12T07:50:21,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742348_1524 (size=12101) 2024-11-12T07:50:21,701 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/d170d352c3924632987aa439e7c3c3b4 2024-11-12T07:50:21,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/8278f07b7d474954a43ba5553aa5ff41 is 50, key is test_row_0/C:col10/1731397820153/Put/seqid=0 2024-11-12T07:50:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T07:50:21,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742349_1525 (size=12101) 2024-11-12T07:50:21,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/8278f07b7d474954a43ba5553aa5ff41 2024-11-12T07:50:21,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/6604837e6f2e49dbbc87044e3ace4c76 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6604837e6f2e49dbbc87044e3ace4c76 2024-11-12T07:50:21,771 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6604837e6f2e49dbbc87044e3ace4c76, entries=150, sequenceid=135, filesize=30.3 K 2024-11-12T07:50:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/d170d352c3924632987aa439e7c3c3b4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/d170d352c3924632987aa439e7c3c3b4 2024-11-12T07:50:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,777 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/d170d352c3924632987aa439e7c3c3b4, entries=150, sequenceid=135, filesize=11.8 K 2024-11-12T07:50:21,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/8278f07b7d474954a43ba5553aa5ff41 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/8278f07b7d474954a43ba5553aa5ff41 2024-11-12T07:50:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,781 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/8278f07b7d474954a43ba5553aa5ff41, entries=150, sequenceid=135, filesize=11.8 K 2024-11-12T07:50:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,782 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 1c3bb6ad593aa45887e06136ab72eae4 in 209ms, sequenceid=135, compaction requested=true 2024-11-12T07:50:21,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:21,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:21,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-12T07:50:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-12T07:50:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-12T07:50:21,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 671 msec 2024-11-12T07:50:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 677 msec 2024-11-12T07:50:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T07:50:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,248 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:50:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-12T07:50:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T07:50:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,253 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:50:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,254 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:50:22,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:50:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T07:50:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:50:22,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T07:50:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:22,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c667df0de447435d8149b665bf56d245_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397822365/Put/seqid=0 2024-11-12T07:50:22,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742350_1526 (size=22268) 2024-11-12T07:50:22,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397822442, max timestamp=1731397822472Num uniques 859; 2024-11-12T07:50:22,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:22,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397822442, max timestamp=1731397822472Num uniques 859; 2024-11-12T07:50:22,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:22,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397822442, max timestamp=1731397822472Num uniques 859; 2024-11-12T07:50:22,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:22,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397882514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397882519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397882520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397882524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T07:50:22,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:22,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:22,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:22,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T07:50:22,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397882628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397882633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397882638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397882638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T07:50:22,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:22,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:22,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:22,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397882838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397882848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397882848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397882858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:22,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:22,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T07:50:22,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:22,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:22,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:22,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:22,875 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T07:50:22,878 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c667df0de447435d8149b665bf56d245_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c667df0de447435d8149b665bf56d245_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:22,879 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/8857f8410b7545e88a53195721320966, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:22,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/8857f8410b7545e88a53195721320966 is 175, key is test_row_0/A:col10/1731397822365/Put/seqid=0 2024-11-12T07:50:22,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742351_1527 (size=65673) 2024-11-12T07:50:22,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/8857f8410b7545e88a53195721320966 2024-11-12T07:50:22,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/01afd0be28e84653bdf3c2b9640b77f2 is 50, key is test_row_0/B:col10/1731397822365/Put/seqid=0 2024-11-12T07:50:22,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742352_1528 (size=12151) 2024-11-12T07:50:22,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/01afd0be28e84653bdf3c2b9640b77f2 2024-11-12T07:50:22,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/328c2013acbb4bdb9dd04bd4d227d97f is 50, key is test_row_0/C:col10/1731397822365/Put/seqid=0 2024-11-12T07:50:22,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742353_1529 (size=12151) 2024-11-12T07:50:23,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T07:50:23,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:23,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397883148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397883158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397883158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397883167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T07:50:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T07:50:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:23,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/328c2013acbb4bdb9dd04bd4d227d97f 2024-11-12T07:50:23,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/8857f8410b7545e88a53195721320966 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8857f8410b7545e88a53195721320966 2024-11-12T07:50:23,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8857f8410b7545e88a53195721320966, entries=350, sequenceid=145, filesize=64.1 K 2024-11-12T07:50:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/01afd0be28e84653bdf3c2b9640b77f2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/01afd0be28e84653bdf3c2b9640b77f2 2024-11-12T07:50:23,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/01afd0be28e84653bdf3c2b9640b77f2, entries=150, sequenceid=145, filesize=11.9 K 2024-11-12T07:50:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/328c2013acbb4bdb9dd04bd4d227d97f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/328c2013acbb4bdb9dd04bd4d227d97f 2024-11-12T07:50:23,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/328c2013acbb4bdb9dd04bd4d227d97f, entries=150, sequenceid=145, filesize=11.9 K 2024-11-12T07:50:23,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1c3bb6ad593aa45887e06136ab72eae4 in 960ms, sequenceid=145, compaction requested=true 2024-11-12T07:50:23,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:23,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:50:23,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:23,352 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:50:23,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:50:23,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:23,352 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:50:23,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:50:23,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:23,353 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:50:23,353 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:50:23,353 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/B is initiating minor compaction (all files) 2024-11-12T07:50:23,353 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/A is initiating minor compaction (all files) 2024-11-12T07:50:23,353 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/A in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,353 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/B in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,353 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e49fe669105c42fea7c0322900a6c362, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4644933fd1ad40f98247a01c8253c365, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6604837e6f2e49dbbc87044e3ace4c76, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8857f8410b7545e88a53195721320966] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=171.9 K 2024-11-12T07:50:23,353 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/7a4f71d4bc0b4265a3d63e47b0a3c7c8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e760044bb0c94c7aaa1d91ead9a87c23, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/d170d352c3924632987aa439e7c3c3b4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/01afd0be28e84653bdf3c2b9640b77f2] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=47.3 K 2024-11-12T07:50:23,353 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,353 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e49fe669105c42fea7c0322900a6c362, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4644933fd1ad40f98247a01c8253c365, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6604837e6f2e49dbbc87044e3ace4c76, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8857f8410b7545e88a53195721320966] 2024-11-12T07:50:23,354 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a4f71d4bc0b4265a3d63e47b0a3c7c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731397818330 2024-11-12T07:50:23,354 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e49fe669105c42fea7c0322900a6c362, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731397818330 2024-11-12T07:50:23,354 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e760044bb0c94c7aaa1d91ead9a87c23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731397819502 2024-11-12T07:50:23,354 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4644933fd1ad40f98247a01c8253c365, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731397819499 2024-11-12T07:50:23,354 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting d170d352c3924632987aa439e7c3c3b4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731397820150 2024-11-12T07:50:23,354 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6604837e6f2e49dbbc87044e3ace4c76, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731397820150 2024-11-12T07:50:23,354 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 01afd0be28e84653bdf3c2b9640b77f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731397822365 2024-11-12T07:50:23,354 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8857f8410b7545e88a53195721320966, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731397822360 2024-11-12T07:50:23,361 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:23,362 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:23,362 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/4f247939d3324299a0bc381dd9c25e91 is 50, key is test_row_0/B:col10/1731397822365/Put/seqid=0 2024-11-12T07:50:23,363 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112ef882f3b545f40d3be83f21ea36893b9_1c3bb6ad593aa45887e06136ab72eae4 store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:23,366 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112ef882f3b545f40d3be83f21ea36893b9_1c3bb6ad593aa45887e06136ab72eae4, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:23,366 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ef882f3b545f40d3be83f21ea36893b9_1c3bb6ad593aa45887e06136ab72eae4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:23,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742355_1531 (size=4469) 2024-11-12T07:50:23,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742354_1530 (size=12493) 2024-11-12T07:50:23,379 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/4f247939d3324299a0bc381dd9c25e91 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/4f247939d3324299a0bc381dd9c25e91 2024-11-12T07:50:23,384 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/B of 1c3bb6ad593aa45887e06136ab72eae4 into 4f247939d3324299a0bc381dd9c25e91(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:23,384 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:23,385 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/B, priority=12, startTime=1731397823352; duration=0sec 2024-11-12T07:50:23,385 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:23,385 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:B 2024-11-12T07:50:23,385 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:50:23,387 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:50:23,387 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/C is initiating minor compaction (all files) 2024-11-12T07:50:23,387 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/C in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,387 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/489f2083d899437f9aa83ddcf7446645, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/8278f07b7d474954a43ba5553aa5ff41, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/328c2013acbb4bdb9dd04bd4d227d97f] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=47.3 K 2024-11-12T07:50:23,387 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 489f2083d899437f9aa83ddcf7446645, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731397818330 2024-11-12T07:50:23,388 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c0fdd7e648b4ab6b71ec6fc6baff4fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731397819502 2024-11-12T07:50:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T07:50:23,388 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8278f07b7d474954a43ba5553aa5ff41, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731397820150 2024-11-12T07:50:23,389 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 328c2013acbb4bdb9dd04bd4d227d97f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731397822365 2024-11-12T07:50:23,395 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#C#compaction#449 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:23,395 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/b40a4ce688b74f0faae04af50f7345f5 is 50, key is test_row_0/C:col10/1731397822365/Put/seqid=0 2024-11-12T07:50:23,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742356_1532 (size=12493) 2024-11-12T07:50:23,407 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/b40a4ce688b74f0faae04af50f7345f5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b40a4ce688b74f0faae04af50f7345f5 2024-11-12T07:50:23,411 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/C of 1c3bb6ad593aa45887e06136ab72eae4 into b40a4ce688b74f0faae04af50f7345f5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:23,411 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:23,411 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/C, priority=12, startTime=1731397823352; duration=0sec 2024-11-12T07:50:23,411 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:23,411 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:C 2024-11-12T07:50:23,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T07:50:23,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:23,483 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T07:50:23,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:23,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:23,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:23,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:23,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:23,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112daf0e33a1d364408bd3b8f024f63ae09_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397822519/Put/seqid=0 2024-11-12T07:50:23,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742357_1533 (size=12304) 2024-11-12T07:50:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:23,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:23,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397883675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397883677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397883683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397883683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,771 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#A#compaction#448 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:23,772 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/df8df564f21c4ea090c82546d785d521 is 175, key is test_row_0/A:col10/1731397822365/Put/seqid=0 2024-11-12T07:50:23,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742358_1534 (size=31447) 2024-11-12T07:50:23,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397883788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,795 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/df8df564f21c4ea090c82546d785d521 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/df8df564f21c4ea090c82546d785d521 2024-11-12T07:50:23,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397883798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397883798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:23,806 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/A of 1c3bb6ad593aa45887e06136ab72eae4 into df8df564f21c4ea090c82546d785d521(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:23,806 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:23,806 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/A, priority=12, startTime=1731397823352; duration=0sec 2024-11-12T07:50:23,807 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:23,807 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:A 2024-11-12T07:50:23,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:23,895 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112daf0e33a1d364408bd3b8f024f63ae09_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112daf0e33a1d364408bd3b8f024f63ae09_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/6b224bbe27d34652b04637a07c6e67a5, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/6b224bbe27d34652b04637a07c6e67a5 is 175, key is test_row_0/A:col10/1731397822519/Put/seqid=0 2024-11-12T07:50:23,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742359_1535 (size=31105) 2024-11-12T07:50:24,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397883998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397884007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397884008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,299 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/6b224bbe27d34652b04637a07c6e67a5 2024-11-12T07:50:24,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/b9ddeabb429a421eb9758ce726a58a74 is 50, key is test_row_0/B:col10/1731397822519/Put/seqid=0 2024-11-12T07:50:24,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742360_1536 (size=12151) 2024-11-12T07:50:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397884308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,312 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/b9ddeabb429a421eb9758ce726a58a74 2024-11-12T07:50:24,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397884318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/522fc602c7d44fe18352a329bce532b1 is 50, key is test_row_0/C:col10/1731397822519/Put/seqid=0 2024-11-12T07:50:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397884318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742361_1537 (size=12151) 2024-11-12T07:50:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T07:50:24,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397884698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,700 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 1c3bb6ad593aa45887e06136ab72eae4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:50:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,732 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/522fc602c7d44fe18352a329bce532b1 2024-11-12T07:50:24,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/6b224bbe27d34652b04637a07c6e67a5 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6b224bbe27d34652b04637a07c6e67a5 2024-11-12T07:50:24,744 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6b224bbe27d34652b04637a07c6e67a5, entries=150, sequenceid=184, filesize=30.4 K 2024-11-12T07:50:24,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/b9ddeabb429a421eb9758ce726a58a74 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/b9ddeabb429a421eb9758ce726a58a74 2024-11-12T07:50:24,749 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/b9ddeabb429a421eb9758ce726a58a74, entries=150, sequenceid=184, filesize=11.9 K 2024-11-12T07:50:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/522fc602c7d44fe18352a329bce532b1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/522fc602c7d44fe18352a329bce532b1 2024-11-12T07:50:24,753 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/522fc602c7d44fe18352a329bce532b1, entries=150, sequenceid=184, filesize=11.9 K 2024-11-12T07:50:24,754 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 1c3bb6ad593aa45887e06136ab72eae4 in 1271ms, sequenceid=184, compaction requested=false 2024-11-12T07:50:24,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:24,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:24,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-12T07:50:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-12T07:50:24,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-12T07:50:24,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5020 sec 2024-11-12T07:50:24,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.5090 sec 2024-11-12T07:50:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:24,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T07:50:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:24,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:24,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d45e24a9ce1f49dcb5d5d2dfe5ae4e8c_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397824819/Put/seqid=0 2024-11-12T07:50:24,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742362_1538 (size=14794) 2024-11-12T07:50:24,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397884854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397884855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397884860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397884968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397884968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397884968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397885178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:25,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397885178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397885178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:25,236 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:25,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d45e24a9ce1f49dcb5d5d2dfe5ae4e8c_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d45e24a9ce1f49dcb5d5d2dfe5ae4e8c_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:25,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e79b29f0774547b1b6a0b8f7812110e0, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:25,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e79b29f0774547b1b6a0b8f7812110e0 is 175, key is test_row_0/A:col10/1731397824819/Put/seqid=0 2024-11-12T07:50:25,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742363_1539 (size=39749) 2024-11-12T07:50:25,264 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e79b29f0774547b1b6a0b8f7812110e0 2024-11-12T07:50:25,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/bceb50daf4bc49e0b2252c5d41013896 is 50, key is test_row_0/B:col10/1731397824819/Put/seqid=0 2024-11-12T07:50:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742364_1540 (size=12151) 2024-11-12T07:50:25,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397885487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397885488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:25,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397885488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:25,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/bceb50daf4bc49e0b2252c5d41013896 2024-11-12T07:50:25,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/a62d602b1e0b4ba78efdcfaaf238324c is 50, key is test_row_0/C:col10/1731397824819/Put/seqid=0 2024-11-12T07:50:25,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742365_1541 (size=12151) 2024-11-12T07:50:25,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/a62d602b1e0b4ba78efdcfaaf238324c 2024-11-12T07:50:25,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e79b29f0774547b1b6a0b8f7812110e0 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e79b29f0774547b1b6a0b8f7812110e0 2024-11-12T07:50:25,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e79b29f0774547b1b6a0b8f7812110e0, entries=200, sequenceid=197, filesize=38.8 K 2024-11-12T07:50:25,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/bceb50daf4bc49e0b2252c5d41013896 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/bceb50daf4bc49e0b2252c5d41013896 2024-11-12T07:50:25,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/bceb50daf4bc49e0b2252c5d41013896, entries=150, sequenceid=197, filesize=11.9 K 2024-11-12T07:50:25,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/a62d602b1e0b4ba78efdcfaaf238324c as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a62d602b1e0b4ba78efdcfaaf238324c 2024-11-12T07:50:25,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a62d602b1e0b4ba78efdcfaaf238324c, entries=150, sequenceid=197, filesize=11.9 K 2024-11-12T07:50:25,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1c3bb6ad593aa45887e06136ab72eae4 in 917ms, sequenceid=197, compaction requested=true 2024-11-12T07:50:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:50:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:25,737 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:50:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:25,737 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:50:25,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:25,740 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:25,740 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:25,740 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/B is initiating minor compaction (all files) 2024-11-12T07:50:25,740 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/A is initiating minor compaction (all files) 2024-11-12T07:50:25,740 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/A in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:25,740 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/B in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:25,740 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/df8df564f21c4ea090c82546d785d521, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6b224bbe27d34652b04637a07c6e67a5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e79b29f0774547b1b6a0b8f7812110e0] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=99.9 K 2024-11-12T07:50:25,740 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/4f247939d3324299a0bc381dd9c25e91, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/b9ddeabb429a421eb9758ce726a58a74, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/bceb50daf4bc49e0b2252c5d41013896] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=35.9 K 2024-11-12T07:50:25,740 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:25,740 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/df8df564f21c4ea090c82546d785d521, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6b224bbe27d34652b04637a07c6e67a5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e79b29f0774547b1b6a0b8f7812110e0] 2024-11-12T07:50:25,740 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f247939d3324299a0bc381dd9c25e91, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731397822365 2024-11-12T07:50:25,740 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting df8df564f21c4ea090c82546d785d521, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731397822365 2024-11-12T07:50:25,741 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ddeabb429a421eb9758ce726a58a74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731397822512 2024-11-12T07:50:25,741 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b224bbe27d34652b04637a07c6e67a5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731397822512 2024-11-12T07:50:25,741 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting bceb50daf4bc49e0b2252c5d41013896, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731397823671 2024-11-12T07:50:25,741 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e79b29f0774547b1b6a0b8f7812110e0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731397823671 2024-11-12T07:50:25,756 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:25,760 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:25,760 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/ee40c761eea74ca0ab2993dea54eb905 is 50, key is test_row_0/B:col10/1731397824819/Put/seqid=0 2024-11-12T07:50:25,763 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111247bf6e6cd5c5457f9fabfcf3b97c6198_1c3bb6ad593aa45887e06136ab72eae4 store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:25,765 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111247bf6e6cd5c5457f9fabfcf3b97c6198_1c3bb6ad593aa45887e06136ab72eae4, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:25,765 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111247bf6e6cd5c5457f9fabfcf3b97c6198_1c3bb6ad593aa45887e06136ab72eae4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:25,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742366_1542 (size=12595) 2024-11-12T07:50:25,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742367_1543 (size=4469) 2024-11-12T07:50:25,787 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#A#compaction#456 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:25,787 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/51c27c1356d7434fa73eba8902e07725 is 175, key is test_row_0/A:col10/1731397824819/Put/seqid=0 2024-11-12T07:50:25,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742368_1544 (size=31549) 2024-11-12T07:50:25,807 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/51c27c1356d7434fa73eba8902e07725 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/51c27c1356d7434fa73eba8902e07725 2024-11-12T07:50:25,812 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/A of 1c3bb6ad593aa45887e06136ab72eae4 into 51c27c1356d7434fa73eba8902e07725(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:25,812 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:25,812 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/A, priority=13, startTime=1731397825737; duration=0sec 2024-11-12T07:50:25,812 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:25,812 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:A 2024-11-12T07:50:25,812 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:25,814 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:25,814 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/C is initiating minor compaction (all files) 2024-11-12T07:50:25,814 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/C in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:25,814 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b40a4ce688b74f0faae04af50f7345f5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/522fc602c7d44fe18352a329bce532b1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a62d602b1e0b4ba78efdcfaaf238324c] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=35.9 K 2024-11-12T07:50:25,815 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40a4ce688b74f0faae04af50f7345f5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731397822365 2024-11-12T07:50:25,815 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 522fc602c7d44fe18352a329bce532b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731397822512 2024-11-12T07:50:25,815 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting a62d602b1e0b4ba78efdcfaaf238324c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731397823671 2024-11-12T07:50:25,834 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#C#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:25,835 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/13b1b17560844e3e8a8468864f005a48 is 50, key is test_row_0/C:col10/1731397824819/Put/seqid=0 2024-11-12T07:50:25,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742369_1545 (size=12595) 2024-11-12T07:50:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:26,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T07:50:26,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397824848, max timestamp=1731397825999Num uniques 859; 2024-11-12T07:50:26,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397824848, max timestamp=1731397825999Num uniques 859; 2024-11-12T07:50:26,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397824848, max timestamp=1731397825999Num uniques 859; 2024-11-12T07:50:26,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:26,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:26,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:26,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:26,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:26,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:26,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:26,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:26,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:26,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b59ead9915fd49139265ed8d3dfc1ef3_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397826000/Put/seqid=0 2024-11-12T07:50:26,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742370_1546 (size=12304) 2024-11-12T07:50:26,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397826027, max timestamp=1731397826040Num uniques 859; 2024-11-12T07:50:26,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397826027, max timestamp=1731397826040Num uniques 859; 2024-11-12T07:50:26,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:26,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:26,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731397826027, max timestamp=1731397826040Num uniques 859; 2024-11-12T07:50:26,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T07:50:26,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397826067, max timestamp=1731397826080Num uniques 1018; 2024-11-12T07:50:26,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397826067, max timestamp=1731397826080Num uniques 1018; 2024-11-12T07:50:26,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731397826067, max timestamp=1731397826080Num uniques 1018; 2024-11-12T07:50:26,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T07:50:26,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T07:50:26,084 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T07:50:26,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731397826111, max timestamp=1731397826121Num uniques 1077; 2024-11-12T07:50:26,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731397826111, max timestamp=1731397826121Num uniques 1077; 2024-11-12T07:50:26,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T07:50:26,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T07:50:26,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731397826111, max timestamp=1731397826121Num uniques 1077; 2024-11-12T07:50:26,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T07:50:26,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397826152, max timestamp=1731397826176Num uniques 1086; 2024-11-12T07:50:26,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T07:50:26,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397826152, max timestamp=1731397826176Num uniques 1086; 2024-11-12T07:50:26,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T07:50:26,182 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/ee40c761eea74ca0ab2993dea54eb905 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/ee40c761eea74ca0ab2993dea54eb905 2024-11-12T07:50:26,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731397826152, max timestamp=1731397826176Num uniques 1086; 2024-11-12T07:50:26,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T07:50:26,188 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/B of 1c3bb6ad593aa45887e06136ab72eae4 into ee40c761eea74ca0ab2993dea54eb905(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:26,188 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:26,188 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/B, priority=13, startTime=1731397825737; duration=0sec 2024-11-12T07:50:26,188 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:26,188 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:B 2024-11-12T07:50:26,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731397826215, max timestamp=1731397826226Num uniques 1045; 2024-11-12T07:50:26,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T07:50:26,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731397826215, max timestamp=1731397826226Num uniques 1045; 2024-11-12T07:50:26,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T07:50:26,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731397826215, max timestamp=1731397826226Num uniques 1045; 2024-11-12T07:50:26,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T07:50:26,258 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/13b1b17560844e3e8a8468864f005a48 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/13b1b17560844e3e8a8468864f005a48 2024-11-12T07:50:26,264 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/C of 1c3bb6ad593aa45887e06136ab72eae4 into 13b1b17560844e3e8a8468864f005a48(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:26,264 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:26,264 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/C, priority=13, startTime=1731397825737; duration=0sec 2024-11-12T07:50:26,265 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:26,265 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:C 2024-11-12T07:50:26,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731397826273, max timestamp=1731397826286Num uniques 1104; 2024-11-12T07:50:26,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=11630, suffix heap size=142784,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=254 2024-11-12T07:50:26,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731397826273, max timestamp=1731397826286Num uniques 1104; 2024-11-12T07:50:26,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-12T07:50:26,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731397826273, max timestamp=1731397826286Num uniques 1104; 2024-11-12T07:50:26,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-12T07:50:26,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731397826330, max timestamp=1731397826354Num uniques 1113; 2024-11-12T07:50:26,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731397826330, max timestamp=1731397826354Num uniques 1013; 2024-11-12T07:50:26,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-11-12T07:50:26,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-12T07:50:26,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731397826343, max timestamp=1731397826354Num uniques 1013; 2024-11-12T07:50:26,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-12T07:50:26,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731397826408, max timestamp=1731397826418Num uniques 1072; 2024-11-12T07:50:26,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-12T07:50:26,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731397826408, max timestamp=1731397826418Num uniques 1072; 2024-11-12T07:50:26,422 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-12T07:50:26,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731397826408, max timestamp=1731397826418Num uniques 1022; 2024-11-12T07:50:26,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-12T07:50:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T07:50:26,428 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:26,429 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:26,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:50:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-12T07:50:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T07:50:26,436 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:50:26,436 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b59ead9915fd49139265ed8d3dfc1ef3_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b59ead9915fd49139265ed8d3dfc1ef3_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:26,437 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:50:26,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:50:26,437 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/dcf8c24c13be4fa9ba976102d08c99e1, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:26,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/dcf8c24c13be4fa9ba976102d08c99e1 is 175, key is test_row_0/A:col10/1731397826000/Put/seqid=0 2024-11-12T07:50:26,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742371_1547 (size=31105) 2024-11-12T07:50:26,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/dcf8c24c13be4fa9ba976102d08c99e1 2024-11-12T07:50:26,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/f27eef36cba1403887aa7f41d60ff9cd is 50, key is test_row_0/B:col10/1731397826000/Put/seqid=0 2024-11-12T07:50:26,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8278, totalHeapSize=19064, min timestamp=1731397826450, max timestamp=1731397826470Num uniques 1081; 2024-11-12T07:50:26,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49498, new segment data size=8278, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-12T07:50:26,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731397826450, max timestamp=1731397826470Num uniques 1081; 2024-11-12T07:50:26,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=8276, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-12T07:50:26,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8276, totalHeapSize=19064, min timestamp=1731397826450, max timestamp=1731397826470Num uniques 1081; 2024-11-12T07:50:26,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49496, new segment data size=8276, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-12T07:50:26,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742372_1548 (size=12151) 2024-11-12T07:50:26,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/f27eef36cba1403887aa7f41d60ff9cd 2024-11-12T07:50:26,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731397826483, max timestamp=1731397826521Num uniques 1040; 2024-11-12T07:50:26,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-12T07:50:26,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731397826483, max timestamp=1731397826521Num uniques 1040; 2024-11-12T07:50:26,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-12T07:50:26,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731397826483, max timestamp=1731397826521Num uniques 1040; 2024-11-12T07:50:26,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-12T07:50:26,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/b274cae46d544b24a87ad0c4de0f3721 is 50, key is test_row_0/C:col10/1731397826000/Put/seqid=0 2024-11-12T07:50:26,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742373_1549 (size=12151) 2024-11-12T07:50:26,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/b274cae46d544b24a87ad0c4de0f3721 2024-11-12T07:50:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T07:50:26,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/dcf8c24c13be4fa9ba976102d08c99e1 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/dcf8c24c13be4fa9ba976102d08c99e1 2024-11-12T07:50:26,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/dcf8c24c13be4fa9ba976102d08c99e1, entries=150, sequenceid=220, filesize=30.4 K 2024-11-12T07:50:26,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/f27eef36cba1403887aa7f41d60ff9cd as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f27eef36cba1403887aa7f41d60ff9cd 2024-11-12T07:50:26,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f27eef36cba1403887aa7f41d60ff9cd, entries=150, sequenceid=220, filesize=11.9 K 2024-11-12T07:50:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/b274cae46d544b24a87ad0c4de0f3721 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b274cae46d544b24a87ad0c4de0f3721 2024-11-12T07:50:26,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b274cae46d544b24a87ad0c4de0f3721, entries=150, sequenceid=220, filesize=11.9 K 2024-11-12T07:50:26,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 1c3bb6ad593aa45887e06136ab72eae4 in 581ms, sequenceid=220, compaction requested=false 2024-11-12T07:50:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:26,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=161.02 KB heapSize=457.78 KB 2024-11-12T07:50:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:50:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:50:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:26,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T07:50:26,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397886588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397886589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f2d034bce3a848a4a7f0f15eef3e4c09_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397826579/Put/seqid=0 2024-11-12T07:50:26,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:26,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397886591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742374_1550 (size=20074) 2024-11-12T07:50:26,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397886698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397886698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397886708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397886728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,731 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 1c3bb6ad593aa45887e06136ab72eae4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:50:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:26,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T07:50:26,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:26,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397886908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397886909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:26,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397886919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:26,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:27,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:27,023 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f2d034bce3a848a4a7f0f15eef3e4c09_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2d034bce3a848a4a7f0f15eef3e4c09_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:27,024 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/deb10096d5a4456eac46619ea68de819, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:27,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/deb10096d5a4456eac46619ea68de819 is 175, key is test_row_0/A:col10/1731397826579/Put/seqid=0 2024-11-12T07:50:27,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742375_1551 (size=57333) 2024-11-12T07:50:27,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T07:50:27,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397887220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:27,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397887220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:27,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397887228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:27,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/deb10096d5a4456eac46619ea68de819 2024-11-12T07:50:27,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/97e7ff3ce0484b0780a6d41ffa6ef159 is 50, key is test_row_0/B:col10/1731397826579/Put/seqid=0 2024-11-12T07:50:27,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742376_1552 (size=12301) 2024-11-12T07:50:27,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:27,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:27,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T07:50:27,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397887727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397887738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:27,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397887738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:27,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:27,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/97e7ff3ce0484b0780a6d41ffa6ef159 2024-11-12T07:50:27,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/e6db49fa3196429681195b82b1c6f31f is 50, key is test_row_0/C:col10/1731397826579/Put/seqid=0 2024-11-12T07:50:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742377_1553 (size=12301) 2024-11-12T07:50:27,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/e6db49fa3196429681195b82b1c6f31f 2024-11-12T07:50:27,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/deb10096d5a4456eac46619ea68de819 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/deb10096d5a4456eac46619ea68de819 2024-11-12T07:50:27,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/deb10096d5a4456eac46619ea68de819, entries=300, sequenceid=416, filesize=56.0 K 2024-11-12T07:50:27,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/97e7ff3ce0484b0780a6d41ffa6ef159 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/97e7ff3ce0484b0780a6d41ffa6ef159 2024-11-12T07:50:27,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/97e7ff3ce0484b0780a6d41ffa6ef159, entries=150, sequenceid=416, filesize=12.0 K 2024-11-12T07:50:27,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/e6db49fa3196429681195b82b1c6f31f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/e6db49fa3196429681195b82b1c6f31f 2024-11-12T07:50:27,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/e6db49fa3196429681195b82b1c6f31f, entries=150, sequenceid=416, filesize=12.0 K 2024-11-12T07:50:27,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~457.73 KB/468720, currentSize=20.13 KB/20610 for 1c3bb6ad593aa45887e06136ab72eae4 in 1340ms, sequenceid=416, compaction requested=true 2024-11-12T07:50:27,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:27,923 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:27,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:50:27,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:27,923 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:27,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:50:27,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:27,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:50:27,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:27,929 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:27,929 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/B is initiating minor compaction (all files) 2024-11-12T07:50:27,929 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/B in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,929 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/ee40c761eea74ca0ab2993dea54eb905, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f27eef36cba1403887aa7f41d60ff9cd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/97e7ff3ce0484b0780a6d41ffa6ef159] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=36.2 K 2024-11-12T07:50:27,929 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119987 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:27,929 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/A is initiating minor compaction (all files) 2024-11-12T07:50:27,929 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/A in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,930 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/51c27c1356d7434fa73eba8902e07725, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/dcf8c24c13be4fa9ba976102d08c99e1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/deb10096d5a4456eac46619ea68de819] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=117.2 K 2024-11-12T07:50:27,930 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,930 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/51c27c1356d7434fa73eba8902e07725, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/dcf8c24c13be4fa9ba976102d08c99e1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/deb10096d5a4456eac46619ea68de819] 2024-11-12T07:50:27,930 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ee40c761eea74ca0ab2993dea54eb905, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731397823671 2024-11-12T07:50:27,931 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51c27c1356d7434fa73eba8902e07725, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731397823671 2024-11-12T07:50:27,931 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f27eef36cba1403887aa7f41d60ff9cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731397824853 2024-11-12T07:50:27,932 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf8c24c13be4fa9ba976102d08c99e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731397824853 2024-11-12T07:50:27,932 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 97e7ff3ce0484b0780a6d41ffa6ef159, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731397826578 2024-11-12T07:50:27,932 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting deb10096d5a4456eac46619ea68de819, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731397826566 2024-11-12T07:50:27,945 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:27,945 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:27,947 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/86b2e45ef5ab40daacdda7f6aba2c986 is 50, key is test_row_0/B:col10/1731397826579/Put/seqid=0 2024-11-12T07:50:27,954 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112dda4c0337b184644b725fa6f6d3b47d4_1c3bb6ad593aa45887e06136ab72eae4 store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:27,956 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112dda4c0337b184644b725fa6f6d3b47d4_1c3bb6ad593aa45887e06136ab72eae4, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:27,957 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dda4c0337b184644b725fa6f6d3b47d4_1c3bb6ad593aa45887e06136ab72eae4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:27,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742378_1554 (size=12847) 2024-11-12T07:50:27,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T07:50:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,977 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T07:50:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:27,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:27,978 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/86b2e45ef5ab40daacdda7f6aba2c986 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/86b2e45ef5ab40daacdda7f6aba2c986 2024-11-12T07:50:27,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742379_1555 (size=4469) 2024-11-12T07:50:27,992 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/B of 1c3bb6ad593aa45887e06136ab72eae4 into 86b2e45ef5ab40daacdda7f6aba2c986(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:27,992 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:27,992 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/B, priority=13, startTime=1731397827923; duration=0sec 2024-11-12T07:50:27,992 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:27,992 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:B 2024-11-12T07:50:27,992 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:27,993 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:27,993 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/C is initiating minor compaction (all files) 2024-11-12T07:50:27,993 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/C in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:27,993 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/13b1b17560844e3e8a8468864f005a48, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b274cae46d544b24a87ad0c4de0f3721, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/e6db49fa3196429681195b82b1c6f31f] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=36.2 K 2024-11-12T07:50:27,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120f9ff2d00d56447e888c4a4aa3c4d550_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397826589/Put/seqid=0 2024-11-12T07:50:27,995 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 13b1b17560844e3e8a8468864f005a48, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731397823671 2024-11-12T07:50:27,999 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting b274cae46d544b24a87ad0c4de0f3721, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731397824853 2024-11-12T07:50:27,999 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e6db49fa3196429681195b82b1c6f31f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731397826578 2024-11-12T07:50:28,018 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#C#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:28,019 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/10ed856d12d34356b071af36a67a1061 is 50, key is test_row_0/C:col10/1731397826579/Put/seqid=0 2024-11-12T07:50:28,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742380_1556 (size=12454) 2024-11-12T07:50:28,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,054 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120f9ff2d00d56447e888c4a4aa3c4d550_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120f9ff2d00d56447e888c4a4aa3c4d550_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/db01f16f8d5d4b7580647149c22cd301, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/db01f16f8d5d4b7580647149c22cd301 is 175, key is test_row_0/A:col10/1731397826589/Put/seqid=0 2024-11-12T07:50:28,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742381_1557 (size=12847) 2024-11-12T07:50:28,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742382_1558 (size=31255) 2024-11-12T07:50:28,117 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/db01f16f8d5d4b7580647149c22cd301 2024-11-12T07:50:28,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/201c1a1f4ef140d7a989e17f3c0d8536 is 50, key is test_row_0/B:col10/1731397826589/Put/seqid=0 2024-11-12T07:50:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742383_1559 (size=12301) 2024-11-12T07:50:28,389 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#A#compaction#466 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:28,389 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/8ca87ebe98f74ae7beb2098cb86fe049 is 175, key is test_row_0/A:col10/1731397826579/Put/seqid=0 2024-11-12T07:50:28,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742384_1560 (size=31801) 2024-11-12T07:50:28,433 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/8ca87ebe98f74ae7beb2098cb86fe049 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8ca87ebe98f74ae7beb2098cb86fe049 2024-11-12T07:50:28,439 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/A of 1c3bb6ad593aa45887e06136ab72eae4 into 8ca87ebe98f74ae7beb2098cb86fe049(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:28,440 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:28,440 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/A, priority=13, startTime=1731397827923; duration=0sec 2024-11-12T07:50:28,440 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:28,440 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:A 2024-11-12T07:50:28,494 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/10ed856d12d34356b071af36a67a1061 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/10ed856d12d34356b071af36a67a1061 2024-11-12T07:50:28,499 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/C of 1c3bb6ad593aa45887e06136ab72eae4 into 10ed856d12d34356b071af36a67a1061(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:28,499 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:28,499 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/C, priority=13, startTime=1731397827924; duration=0sec 2024-11-12T07:50:28,499 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:28,499 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:C 2024-11-12T07:50:28,538 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/201c1a1f4ef140d7a989e17f3c0d8536 2024-11-12T07:50:28,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/a05ce3bb6b0b49b9912aaeb1c8762533 is 50, key is test_row_0/C:col10/1731397826589/Put/seqid=0 2024-11-12T07:50:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742385_1561 (size=12301) 2024-11-12T07:50:28,570 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/a05ce3bb6b0b49b9912aaeb1c8762533 2024-11-12T07:50:28,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/db01f16f8d5d4b7580647149c22cd301 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/db01f16f8d5d4b7580647149c22cd301 2024-11-12T07:50:28,582 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/db01f16f8d5d4b7580647149c22cd301, entries=150, sequenceid=422, filesize=30.5 K 2024-11-12T07:50:28,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/201c1a1f4ef140d7a989e17f3c0d8536 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/201c1a1f4ef140d7a989e17f3c0d8536 2024-11-12T07:50:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T07:50:28,593 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/201c1a1f4ef140d7a989e17f3c0d8536, entries=150, sequenceid=422, filesize=12.0 K 2024-11-12T07:50:28,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/a05ce3bb6b0b49b9912aaeb1c8762533 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a05ce3bb6b0b49b9912aaeb1c8762533 2024-11-12T07:50:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,604 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a05ce3bb6b0b49b9912aaeb1c8762533, entries=150, sequenceid=422, filesize=12.0 K 2024-11-12T07:50:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,606 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 1c3bb6ad593aa45887e06136ab72eae4 in 629ms, sequenceid=422, compaction requested=false 2024-11-12T07:50:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:28,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-12T07:50:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-12T07:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-12T07:50:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1720 sec 2024-11-12T07:50:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.1840 sec 2024-11-12T07:50:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:28,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:50:28,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:28,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:28,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:28,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:28,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:28,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111270c50aeddb0f47c29814c43f87c8573c_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397828766/Put/seqid=0 2024-11-12T07:50:28,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397828842, max timestamp=1731397828857Num uniques 859; 2024-11-12T07:50:28,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397828842, max timestamp=1731397828857Num uniques 859; 2024-11-12T07:50:28,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:50:28,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:50:28,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742386_1562 (size=12454) 2024-11-12T07:50:28,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397888889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397888890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:28,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397888891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:28,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397888998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397889007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397889007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,130 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T07:50:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397889208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397889217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397889218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,270 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:29,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111270c50aeddb0f47c29814c43f87c8573c_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111270c50aeddb0f47c29814c43f87c8573c_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:29,276 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/eed7ca7aa04f4836b9ff43be35f5204d, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:29,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/eed7ca7aa04f4836b9ff43be35f5204d is 175, key is test_row_0/A:col10/1731397828766/Put/seqid=0 2024-11-12T07:50:29,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742387_1563 (size=31255) 2024-11-12T07:50:29,282 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/eed7ca7aa04f4836b9ff43be35f5204d 2024-11-12T07:50:29,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/048ade6a02684896892a8380f41a9063 is 50, key is test_row_0/B:col10/1731397828766/Put/seqid=0 2024-11-12T07:50:29,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742388_1564 (size=12301) 2024-11-12T07:50:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397889518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397889528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397889528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:29,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/048ade6a02684896892a8380f41a9063 2024-11-12T07:50:29,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/95ac32ef83ad4b64a5a27a50ec8c12b2 is 50, key is test_row_0/C:col10/1731397828766/Put/seqid=0 2024-11-12T07:50:29,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742389_1565 (size=12301) 2024-11-12T07:50:29,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/95ac32ef83ad4b64a5a27a50ec8c12b2 2024-11-12T07:50:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/eed7ca7aa04f4836b9ff43be35f5204d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/eed7ca7aa04f4836b9ff43be35f5204d 2024-11-12T07:50:29,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/eed7ca7aa04f4836b9ff43be35f5204d, entries=150, sequenceid=435, filesize=30.5 K 2024-11-12T07:50:29,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/048ade6a02684896892a8380f41a9063 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/048ade6a02684896892a8380f41a9063 2024-11-12T07:50:29,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/048ade6a02684896892a8380f41a9063, entries=150, sequenceid=435, filesize=12.0 K 2024-11-12T07:50:29,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/95ac32ef83ad4b64a5a27a50ec8c12b2 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/95ac32ef83ad4b64a5a27a50ec8c12b2 2024-11-12T07:50:29,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/95ac32ef83ad4b64a5a27a50ec8c12b2, entries=150, sequenceid=435, filesize=12.0 K 2024-11-12T07:50:29,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for 1c3bb6ad593aa45887e06136ab72eae4 in 1021ms, sequenceid=435, compaction requested=true 2024-11-12T07:50:29,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:29,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:50:29,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:29,816 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:29,816 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:29,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:50:29,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:29,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:50:29,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:29,817 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:29,817 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/B is initiating minor compaction (all files) 2024-11-12T07:50:29,817 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/B in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:29,817 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/86b2e45ef5ab40daacdda7f6aba2c986, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/201c1a1f4ef140d7a989e17f3c0d8536, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/048ade6a02684896892a8380f41a9063] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=36.6 K 2024-11-12T07:50:29,817 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:29,817 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/A is initiating minor compaction (all files) 2024-11-12T07:50:29,817 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/A in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:29,818 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8ca87ebe98f74ae7beb2098cb86fe049, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/db01f16f8d5d4b7580647149c22cd301, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/eed7ca7aa04f4836b9ff43be35f5204d] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=92.1 K 2024-11-12T07:50:29,818 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:29,818 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8ca87ebe98f74ae7beb2098cb86fe049, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/db01f16f8d5d4b7580647149c22cd301, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/eed7ca7aa04f4836b9ff43be35f5204d] 2024-11-12T07:50:29,818 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b2e45ef5ab40daacdda7f6aba2c986, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731397826578 2024-11-12T07:50:29,818 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ca87ebe98f74ae7beb2098cb86fe049, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731397826578 2024-11-12T07:50:29,818 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 201c1a1f4ef140d7a989e17f3c0d8536, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731397826586 2024-11-12T07:50:29,818 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting db01f16f8d5d4b7580647149c22cd301, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731397826586 2024-11-12T07:50:29,818 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 048ade6a02684896892a8380f41a9063, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397828766 2024-11-12T07:50:29,818 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed7ca7aa04f4836b9ff43be35f5204d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397828766 2024-11-12T07:50:29,828 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:29,829 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/8943f496ecc44e279c5fdd51f1dc4d8e is 50, key is test_row_0/B:col10/1731397828766/Put/seqid=0 2024-11-12T07:50:29,835 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:29,838 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112c6eb6e70618d4e698fe3a898d6415467_1c3bb6ad593aa45887e06136ab72eae4 store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:29,840 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112c6eb6e70618d4e698fe3a898d6415467_1c3bb6ad593aa45887e06136ab72eae4, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:29,840 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c6eb6e70618d4e698fe3a898d6415467_1c3bb6ad593aa45887e06136ab72eae4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:29,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742390_1566 (size=12949) 2024-11-12T07:50:29,847 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/8943f496ecc44e279c5fdd51f1dc4d8e as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/8943f496ecc44e279c5fdd51f1dc4d8e 2024-11-12T07:50:29,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742391_1567 (size=4469) 2024-11-12T07:50:29,853 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/B of 1c3bb6ad593aa45887e06136ab72eae4 into 8943f496ecc44e279c5fdd51f1dc4d8e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:29,853 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:29,854 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/B, priority=13, startTime=1731397829816; duration=0sec 2024-11-12T07:50:29,854 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:29,854 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:B 2024-11-12T07:50:29,854 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T07:50:29,855 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#A#compaction#475 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:29,855 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/97365be1ccfe4b28ad3ec1061e96a13a is 175, key is test_row_0/A:col10/1731397828766/Put/seqid=0 2024-11-12T07:50:29,857 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T07:50:29,857 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/C is initiating minor compaction (all files) 2024-11-12T07:50:29,857 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/C in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:29,858 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/10ed856d12d34356b071af36a67a1061, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a05ce3bb6b0b49b9912aaeb1c8762533, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/95ac32ef83ad4b64a5a27a50ec8c12b2] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=36.6 K 2024-11-12T07:50:29,858 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 10ed856d12d34356b071af36a67a1061, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731397826578 2024-11-12T07:50:29,858 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting a05ce3bb6b0b49b9912aaeb1c8762533, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731397826586 2024-11-12T07:50:29,859 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ac32ef83ad4b64a5a27a50ec8c12b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397828766 2024-11-12T07:50:29,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742392_1568 (size=31903) 2024-11-12T07:50:29,886 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/97365be1ccfe4b28ad3ec1061e96a13a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/97365be1ccfe4b28ad3ec1061e96a13a 2024-11-12T07:50:29,892 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/A of 1c3bb6ad593aa45887e06136ab72eae4 into 97365be1ccfe4b28ad3ec1061e96a13a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:29,892 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:29,892 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/A, priority=13, startTime=1731397829816; duration=0sec 2024-11-12T07:50:29,892 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:29,892 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:A 2024-11-12T07:50:29,896 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:29,896 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/ba503536890e497aa1f02a4446bcbbd4 is 50, key is test_row_0/C:col10/1731397828766/Put/seqid=0 2024-11-12T07:50:29,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742393_1569 (size=12949) 2024-11-12T07:50:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:30,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=136.42 KB heapSize=393.47 KB 2024-11-12T07:50:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:30,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:30,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112001220291e114620b1b2c977147b00f9_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397830029/Put/seqid=0 2024-11-12T07:50:30,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742394_1570 (size=14994) 2024-11-12T07:50:30,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,045 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112001220291e114620b1b2c977147b00f9_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112001220291e114620b1b2c977147b00f9_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:30,046 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/548f8d0e44d64bf0b4e2d60293e9b645, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:30,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/548f8d0e44d64bf0b4e2d60293e9b645 is 175, key is test_row_0/A:col10/1731397830029/Put/seqid=0 2024-11-12T07:50:30,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742395_1571 (size=39949) 2024-11-12T07:50:30,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397890046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397890048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397890048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397890158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397890158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397890159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,309 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/ba503536890e497aa1f02a4446bcbbd4 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ba503536890e497aa1f02a4446bcbbd4 2024-11-12T07:50:30,316 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/C of 1c3bb6ad593aa45887e06136ab72eae4 into ba503536890e497aa1f02a4446bcbbd4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:30,316 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:30,316 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/C, priority=13, startTime=1731397829816; duration=0sec 2024-11-12T07:50:30,316 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:30,316 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:C 2024-11-12T07:50:30,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397890368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397890368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397890368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,450 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=469, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/548f8d0e44d64bf0b4e2d60293e9b645 2024-11-12T07:50:30,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e42a3bebc05a43ecaa10ad9b2ca03d6b is 50, key is test_row_0/B:col10/1731397830029/Put/seqid=0 2024-11-12T07:50:30,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742396_1572 (size=12301) 2024-11-12T07:50:30,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e42a3bebc05a43ecaa10ad9b2ca03d6b 2024-11-12T07:50:30,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/9197ce453e344264b3ebf37713138e9b is 50, key is test_row_0/C:col10/1731397830029/Put/seqid=0 2024-11-12T07:50:30,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742397_1573 (size=12301) 2024-11-12T07:50:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T07:50:30,598 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:30,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:50:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-12T07:50:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T07:50:30,600 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:50:30,600 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:50:30,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:50:30,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397890678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397890678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397890678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T07:50:30,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54240 deadline: 1731397890748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:30,750 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f9fc96254ef8,44299,1731397650797 for 'test_row_2' in 1c3bb6ad593aa45887e06136ab72eae4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-12T07:50:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:30,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:30,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T07:50:30,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:30,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:30,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:30,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:30,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:30,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T07:50:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:30,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:30,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/9197ce453e344264b3ebf37713138e9b 2024-11-12T07:50:30,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/548f8d0e44d64bf0b4e2d60293e9b645 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/548f8d0e44d64bf0b4e2d60293e9b645 2024-11-12T07:50:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T07:50:30,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/548f8d0e44d64bf0b4e2d60293e9b645, entries=200, sequenceid=469, filesize=39.0 K 2024-11-12T07:50:30,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/e42a3bebc05a43ecaa10ad9b2ca03d6b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e42a3bebc05a43ecaa10ad9b2ca03d6b 2024-11-12T07:50:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e42a3bebc05a43ecaa10ad9b2ca03d6b, entries=150, sequenceid=469, filesize=12.0 K 2024-11-12T07:50:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/9197ce453e344264b3ebf37713138e9b as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9197ce453e344264b3ebf37713138e9b 2024-11-12T07:50:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9197ce453e344264b3ebf37713138e9b, entries=150, sequenceid=469, filesize=12.0 K 2024-11-12T07:50:30,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~393.42 KB/402864, currentSize=46.96 KB/48090 for 1c3bb6ad593aa45887e06136ab72eae4 in 905ms, sequenceid=469, compaction requested=false 2024-11-12T07:50:30,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T07:50:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:31,059 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T07:50:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121a2025eaa2d746fda578cae610148e82_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397830045/Put/seqid=0 2024-11-12T07:50:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742398_1574 (size=9914) 2024-11-12T07:50:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,077 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121a2025eaa2d746fda578cae610148e82_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121a2025eaa2d746fda578cae610148e82_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e0998ff8c1944828b81645ef0b8d7656, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e0998ff8c1944828b81645ef0b8d7656 is 175, key is test_row_0/A:col10/1731397830045/Put/seqid=0 2024-11-12T07:50:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742399_1575 (size=22561) 2024-11-12T07:50:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:31,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397831203, max timestamp=1731397831213Num uniques 859; 2024-11-12T07:50:31,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:50:31,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397831203, max timestamp=1731397831214Num uniques 859; 2024-11-12T07:50:31,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:50:31,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731397831203, max timestamp=1731397831214Num uniques 859; 2024-11-12T07:50:31,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f9fc96254ef8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T07:50:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T07:50:31,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397891244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397891247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397891249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397891252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397891358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397891368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397891368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397891368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,485 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=480, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e0998ff8c1944828b81645ef0b8d7656 2024-11-12T07:50:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/f002ab61eb48482cb9aeacf02f289013 is 50, key is test_row_0/B:col10/1731397830045/Put/seqid=0 2024-11-12T07:50:31,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742400_1576 (size=9857) 2024-11-12T07:50:31,501 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/f002ab61eb48482cb9aeacf02f289013 2024-11-12T07:50:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/6c7d9fb428ed4ca5a549a20c09f92f82 is 50, key is test_row_0/C:col10/1731397830045/Put/seqid=0 2024-11-12T07:50:31,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742401_1577 (size=9857) 2024-11-12T07:50:31,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397891568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397891578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397891578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397891580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T07:50:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397891888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397891888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397891888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397891888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:31,927 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/6c7d9fb428ed4ca5a549a20c09f92f82 2024-11-12T07:50:31,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/e0998ff8c1944828b81645ef0b8d7656 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e0998ff8c1944828b81645ef0b8d7656 2024-11-12T07:50:31,932 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e0998ff8c1944828b81645ef0b8d7656, entries=100, sequenceid=480, filesize=22.0 K 2024-11-12T07:50:31,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/f002ab61eb48482cb9aeacf02f289013 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f002ab61eb48482cb9aeacf02f289013 2024-11-12T07:50:31,937 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f002ab61eb48482cb9aeacf02f289013, entries=100, sequenceid=480, filesize=9.6 K 2024-11-12T07:50:31,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/6c7d9fb428ed4ca5a549a20c09f92f82 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/6c7d9fb428ed4ca5a549a20c09f92f82 2024-11-12T07:50:31,946 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/6c7d9fb428ed4ca5a549a20c09f92f82, entries=100, sequenceid=480, filesize=9.6 K 2024-11-12T07:50:31,947 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1c3bb6ad593aa45887e06136ab72eae4 in 888ms, sequenceid=480, compaction requested=true 2024-11-12T07:50:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-12T07:50:31,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-12T07:50:31,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-12T07:50:31,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3490 sec 2024-11-12T07:50:31,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.3530 sec 2024-11-12T07:50:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:32,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-12T07:50:32,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:32,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:32,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:32,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T07:50:32,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112988ba9e44c2e42ee97629ce6577ce3cb_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397831234/Put/seqid=0 2024-11-12T07:50:32,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742402_1578 (size=14994) 2024-11-12T07:50:32,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397892413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112988ba9e44c2e42ee97629ce6577ce3cb_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112988ba9e44c2e42ee97629ce6577ce3cb_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397892414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397892415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,420 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/c5acb9002575413eb2ff1319b7ebbc44, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/c5acb9002575413eb2ff1319b7ebbc44 is 175, key is test_row_0/A:col10/1731397831234/Put/seqid=0 2024-11-12T07:50:32,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397892418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742403_1579 (size=39949) 2024-11-12T07:50:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397892528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397892528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397892528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397892528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397892738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397892738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397892738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397892738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T07:50:32,748 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:32,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-12T07:50:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-12T07:50:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T07:50:32,751 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T07:50:32,751 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T07:50:32,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T07:50:32,779 INFO [Thread-2222 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2222. 2024-11-12T07:50:32,779 DEBUG [Thread-2222 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:32,779 DEBUG [Thread-2222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,779 DEBUG [Thread-2222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:32,781 INFO [Thread-2224 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2224. 2024-11-12T07:50:32,781 DEBUG [Thread-2224 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:32,781 DEBUG [Thread-2224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,781 DEBUG [Thread-2224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:32,781 INFO [Thread-2220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2220. 2024-11-12T07:50:32,781 DEBUG [Thread-2220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:32,781 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,781 DEBUG [Thread-2220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:32,784 INFO [Thread-2218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2218. 2024-11-12T07:50:32,784 DEBUG [Thread-2218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:32,784 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,784 DEBUG [Thread-2218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:32,784 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-11-12T07:50:32,784 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:32,784 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,785 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:32,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:32,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=517, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/c5acb9002575413eb2ff1319b7ebbc44 2024-11-12T07:50:32,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/116c8e5a3d254ff18f52e8fbc705f7ed is 50, key is test_row_0/B:col10/1731397831234/Put/seqid=0 2024-11-12T07:50:32,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742404_1580 (size=12301) 2024-11-12T07:50:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T07:50:32,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T07:50:32,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:32,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:32,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397893058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397893059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397893059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397893059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:33,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T07:50:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:33,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:33,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:33,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T07:50:33,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T07:50:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/116c8e5a3d254ff18f52e8fbc705f7ed 2024-11-12T07:50:33,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/5e607849e9174612a1556dca88ad2fe7 is 50, key is test_row_0/C:col10/1731397831234/Put/seqid=0 2024-11-12T07:50:33,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742405_1581 (size=12301) 2024-11-12T07:50:33,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T07:50:33,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:33,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T07:50:33,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T07:50:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:33,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:33,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54246 deadline: 1731397893569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54278 deadline: 1731397893568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731397893569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:33,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T07:50:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731397893569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 , the old value is region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T07:50:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., hostname=f9fc96254ef8,44299,1731397650797, seqNum=5 because the exception is null or not the one we care about 2024-11-12T07:50:33,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/5e607849e9174612a1556dca88ad2fe7 2024-11-12T07:50:33,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/c5acb9002575413eb2ff1319b7ebbc44 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/c5acb9002575413eb2ff1319b7ebbc44 2024-11-12T07:50:33,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/c5acb9002575413eb2ff1319b7ebbc44, entries=200, sequenceid=517, filesize=39.0 K 2024-11-12T07:50:33,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/116c8e5a3d254ff18f52e8fbc705f7ed as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/116c8e5a3d254ff18f52e8fbc705f7ed 2024-11-12T07:50:33,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/116c8e5a3d254ff18f52e8fbc705f7ed, entries=150, sequenceid=517, filesize=12.0 K 2024-11-12T07:50:33,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/5e607849e9174612a1556dca88ad2fe7 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/5e607849e9174612a1556dca88ad2fe7 2024-11-12T07:50:33,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T07:50:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T07:50:33,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/5e607849e9174612a1556dca88ad2fe7, entries=150, sequenceid=517, filesize=12.0 K 2024-11-12T07:50:33,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 1c3bb6ad593aa45887e06136ab72eae4 in 1279ms, sequenceid=517, compaction requested=true 2024-11-12T07:50:33,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:33,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T07:50:33,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:33,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T07:50:33,683 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:50:33,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:33,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c3bb6ad593aa45887e06136ab72eae4:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T07:50:33,683 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:50:33,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:33,684 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:50:33,684 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:50:33,684 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/A is initiating minor compaction (all files) 2024-11-12T07:50:33,684 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/B is initiating minor compaction (all files) 2024-11-12T07:50:33,684 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/A in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,684 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/B in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,685 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/8943f496ecc44e279c5fdd51f1dc4d8e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e42a3bebc05a43ecaa10ad9b2ca03d6b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f002ab61eb48482cb9aeacf02f289013, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/116c8e5a3d254ff18f52e8fbc705f7ed] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=46.3 K 2024-11-12T07:50:33,685 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/97365be1ccfe4b28ad3ec1061e96a13a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/548f8d0e44d64bf0b4e2d60293e9b645, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e0998ff8c1944828b81645ef0b8d7656, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/c5acb9002575413eb2ff1319b7ebbc44] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=131.2 K 2024-11-12T07:50:33,685 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,685 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. files: [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/97365be1ccfe4b28ad3ec1061e96a13a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/548f8d0e44d64bf0b4e2d60293e9b645, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e0998ff8c1944828b81645ef0b8d7656, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/c5acb9002575413eb2ff1319b7ebbc44] 2024-11-12T07:50:33,685 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 8943f496ecc44e279c5fdd51f1dc4d8e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397828766 2024-11-12T07:50:33,685 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97365be1ccfe4b28ad3ec1061e96a13a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397828766 2024-11-12T07:50:33,686 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting e42a3bebc05a43ecaa10ad9b2ca03d6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731397828875 2024-11-12T07:50:33,686 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 548f8d0e44d64bf0b4e2d60293e9b645, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731397828875 2024-11-12T07:50:33,686 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting f002ab61eb48482cb9aeacf02f289013, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731397830045 2024-11-12T07:50:33,686 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0998ff8c1944828b81645ef0b8d7656, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731397830045 2024-11-12T07:50:33,686 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 116c8e5a3d254ff18f52e8fbc705f7ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731397831234 2024-11-12T07:50:33,687 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5acb9002575413eb2ff1319b7ebbc44, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731397831234 2024-11-12T07:50:33,692 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T07:50:33,693 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/885933439aaf40e7892fc72882ba15f3 is 50, key is test_row_0/B:col10/1731397831234/Put/seqid=0 2024-11-12T07:50:33,694 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:33,695 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112d3d79ca6d3504a54840c12977708364e_1c3bb6ad593aa45887e06136ab72eae4 store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:33,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742406_1582 (size=13085) 2024-11-12T07:50:33,700 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112d3d79ca6d3504a54840c12977708364e_1c3bb6ad593aa45887e06136ab72eae4, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:33,700 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d3d79ca6d3504a54840c12977708364e_1c3bb6ad593aa45887e06136ab72eae4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742407_1583 (size=4469) 2024-11-12T07:50:33,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44299 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T07:50:33,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:33,832 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-12T07:50:33,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:33,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:33,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:33,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:33,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:33,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:33,843 DEBUG [regionserver/f9fc96254ef8:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-12T07:50:33,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f2fd7c6f3e334de5a5bea60b7686311f_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397832415/Put/seqid=0 2024-11-12T07:50:33,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742408_1584 (size=12454) 2024-11-12T07:50:33,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:33,850 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f2fd7c6f3e334de5a5bea60b7686311f_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2fd7c6f3e334de5a5bea60b7686311f_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/ccb96426326c4aae9074a8fbe74bbf3f, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:33,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/ccb96426326c4aae9074a8fbe74bbf3f is 175, key is test_row_0/A:col10/1731397832415/Put/seqid=0 2024-11-12T07:50:33,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742409_1585 (size=31255) 2024-11-12T07:50:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T07:50:34,105 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T07:50:34,105 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/885933439aaf40e7892fc72882ba15f3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/885933439aaf40e7892fc72882ba15f3 2024-11-12T07:50:34,106 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/b059d470a5f44589b4fbd32a1ba82b16 is 175, key is test_row_0/A:col10/1731397831234/Put/seqid=0 2024-11-12T07:50:34,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742410_1586 (size=32039) 2024-11-12T07:50:34,111 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/B of 1c3bb6ad593aa45887e06136ab72eae4 into 885933439aaf40e7892fc72882ba15f3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:34,111 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:34,111 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/B, priority=12, startTime=1731397833683; duration=0sec 2024-11-12T07:50:34,111 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T07:50:34,111 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:B 2024-11-12T07:50:34,111 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T07:50:34,112 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T07:50:34,113 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1541): 1c3bb6ad593aa45887e06136ab72eae4/C is initiating minor compaction (all files) 2024-11-12T07:50:34,113 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c3bb6ad593aa45887e06136ab72eae4/C in TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:34,113 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ba503536890e497aa1f02a4446bcbbd4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9197ce453e344264b3ebf37713138e9b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/6c7d9fb428ed4ca5a549a20c09f92f82, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/5e607849e9174612a1556dca88ad2fe7] into tmpdir=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp, totalSize=46.3 K 2024-11-12T07:50:34,113 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting ba503536890e497aa1f02a4446bcbbd4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731397828766 2024-11-12T07:50:34,113 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 9197ce453e344264b3ebf37713138e9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731397828875 2024-11-12T07:50:34,114 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c7d9fb428ed4ca5a549a20c09f92f82, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731397830045 2024-11-12T07:50:34,114 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/b059d470a5f44589b4fbd32a1ba82b16 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/b059d470a5f44589b4fbd32a1ba82b16 2024-11-12T07:50:34,114 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e607849e9174612a1556dca88ad2fe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731397831234 2024-11-12T07:50:34,119 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/A of 1c3bb6ad593aa45887e06136ab72eae4 into b059d470a5f44589b4fbd32a1ba82b16(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:34,119 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:34,119 INFO [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/A, priority=12, startTime=1731397833683; duration=0sec 2024-11-12T07:50:34,119 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:34,119 DEBUG [RS:0;f9fc96254ef8:44299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:A 2024-11-12T07:50:34,122 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c3bb6ad593aa45887e06136ab72eae4#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-12T07:50:34,123 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/3b323c1c738b411e983c692ed7574e44 is 50, key is test_row_0/C:col10/1731397831234/Put/seqid=0 2024-11-12T07:50:34,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742411_1587 (size=13085) 2024-11-12T07:50:34,255 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=526, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/ccb96426326c4aae9074a8fbe74bbf3f 2024-11-12T07:50:34,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/adf4342efb974ff4aa2761b82288d68d is 50, key is test_row_0/B:col10/1731397832415/Put/seqid=0 2024-11-12T07:50:34,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742412_1588 (size=12301) 2024-11-12T07:50:34,534 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/3b323c1c738b411e983c692ed7574e44 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3b323c1c738b411e983c692ed7574e44 2024-11-12T07:50:34,541 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1c3bb6ad593aa45887e06136ab72eae4/C of 1c3bb6ad593aa45887e06136ab72eae4 into 3b323c1c738b411e983c692ed7574e44(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T07:50:34,541 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:34,541 INFO [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., storeName=1c3bb6ad593aa45887e06136ab72eae4/C, priority=12, startTime=1731397833683; duration=0sec 2024-11-12T07:50:34,541 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T07:50:34,541 DEBUG [RS:0;f9fc96254ef8:44299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c3bb6ad593aa45887e06136ab72eae4:C 2024-11-12T07:50:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44299 {}] regionserver.HRegion(8855): Flush requested on 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:34,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. as already flushing 2024-11-12T07:50:34,579 INFO [Thread-2205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2205. 2024-11-12T07:50:34,579 DEBUG [Thread-2205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:34,579 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:34,579 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:34,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:34,580 INFO [Thread-2213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2213. 2024-11-12T07:50:34,580 DEBUG [Thread-2213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:34,580 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:34,580 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:34,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:34,589 INFO [Thread-2211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2211. 2024-11-12T07:50:34,589 INFO [Thread-2207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2207. 2024-11-12T07:50:34,589 DEBUG [Thread-2211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:34,589 DEBUG [Thread-2207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:34,589 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:34,589 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:34,590 DEBUG [Thread-2207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:34,590 DEBUG [Thread-2211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:34,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:34,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:34,671 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/adf4342efb974ff4aa2761b82288d68d 2024-11-12T07:50:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/b0392645cb2b451d98f1cad92f358148 is 50, key is test_row_0/C:col10/1731397832415/Put/seqid=0 2024-11-12T07:50:34,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742413_1589 (size=12301) 2024-11-12T07:50:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T07:50:35,088 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/b0392645cb2b451d98f1cad92f358148 2024-11-12T07:50:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/ccb96426326c4aae9074a8fbe74bbf3f as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/ccb96426326c4aae9074a8fbe74bbf3f 2024-11-12T07:50:35,101 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/ccb96426326c4aae9074a8fbe74bbf3f, entries=150, sequenceid=526, filesize=30.5 K 2024-11-12T07:50:35,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/adf4342efb974ff4aa2761b82288d68d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/adf4342efb974ff4aa2761b82288d68d 2024-11-12T07:50:35,106 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/adf4342efb974ff4aa2761b82288d68d, entries=150, sequenceid=526, filesize=12.0 K 2024-11-12T07:50:35,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/b0392645cb2b451d98f1cad92f358148 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b0392645cb2b451d98f1cad92f358148 2024-11-12T07:50:35,109 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b0392645cb2b451d98f1cad92f358148, entries=150, sequenceid=526, filesize=12.0 K 2024-11-12T07:50:35,110 INFO [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=26.84 KB/27480 for 1c3bb6ad593aa45887e06136ab72eae4 in 1278ms, sequenceid=526, compaction requested=false 2024-11-12T07:50:35,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1c3bb6ad593aa45887e06136ab72eae4: 2024-11-12T07:50:35,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:35,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f9fc96254ef8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-12T07:50:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-12T07:50:35,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-12T07:50:35,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3600 sec 2024-11-12T07:50:35,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.3640 sec 2024-11-12T07:50:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T07:50:36,929 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:40,780 INFO [Thread-2209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2209. 2024-11-12T07:50:40,780 DEBUG [Thread-2209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T07:50:40,780 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:40,780 DEBUG [Thread-2209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:40,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2791 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2767 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2803 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2794 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2763 2024-11-12T07:50:40,780 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T07:50:40,780 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:50:40,780 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:40,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:40,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:40,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:40,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-12T07:50:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T07:50:40,784 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397840784"}]},"ts":"1731397840784"} 2024-11-12T07:50:40,786 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T07:50:40,786 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T07:50:40,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T07:50:40,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, UNASSIGN}] 2024-11-12T07:50:40,788 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, UNASSIGN 2024-11-12T07:50:40,788 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=1c3bb6ad593aa45887e06136ab72eae4, regionState=CLOSING, regionLocation=f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:40,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, UNASSIGN because future has completed 2024-11-12T07:50:40,789 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T07:50:40,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797}] 2024-11-12T07:50:40,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T07:50:40,941 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:40,941 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T07:50:40,941 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 1c3bb6ad593aa45887e06136ab72eae4, disabling compactions & flushes 2024-11-12T07:50:40,941 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:40,941 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:40,941 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. after waiting 0 ms 2024-11-12T07:50:40,941 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:40,941 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 1c3bb6ad593aa45887e06136ab72eae4 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T07:50:40,941 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=A 2024-11-12T07:50:40,942 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:40,942 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=B 2024-11-12T07:50:40,942 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:40,942 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1c3bb6ad593aa45887e06136ab72eae4, store=C 2024-11-12T07:50:40,942 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T07:50:40,945 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111241212d362d254c69b29993f4d8111dae_1c3bb6ad593aa45887e06136ab72eae4 is 50, key is test_row_0/A:col10/1731397834588/Put/seqid=0 2024-11-12T07:50:40,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742414_1590 (size=12454) 2024-11-12T07:50:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T07:50:41,349 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T07:50:41,352 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111241212d362d254c69b29993f4d8111dae_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111241212d362d254c69b29993f4d8111dae_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:41,353 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/05ca74c174cc4458b4e66ede929cd409, store: [table=TestAcidGuarantees family=A region=1c3bb6ad593aa45887e06136ab72eae4] 2024-11-12T07:50:41,353 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/05ca74c174cc4458b4e66ede929cd409 is 175, key is test_row_0/A:col10/1731397834588/Put/seqid=0 2024-11-12T07:50:41,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742415_1591 (size=31255) 2024-11-12T07:50:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T07:50:41,756 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=537, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/05ca74c174cc4458b4e66ede929cd409 2024-11-12T07:50:41,799 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/5fe4e16a9ef944d5825ac5d29ee6ec30 is 50, key is test_row_0/B:col10/1731397834588/Put/seqid=0 2024-11-12T07:50:41,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742416_1592 (size=12301) 2024-11-12T07:50:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T07:50:42,203 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/5fe4e16a9ef944d5825ac5d29ee6ec30 2024-11-12T07:50:42,208 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/3958c8ea8ccf49fea243d396808aa152 is 50, key is test_row_0/C:col10/1731397834588/Put/seqid=0 2024-11-12T07:50:42,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742417_1593 (size=12301) 2024-11-12T07:50:42,611 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/3958c8ea8ccf49fea243d396808aa152 2024-11-12T07:50:42,615 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/A/05ca74c174cc4458b4e66ede929cd409 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/05ca74c174cc4458b4e66ede929cd409 2024-11-12T07:50:42,618 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/05ca74c174cc4458b4e66ede929cd409, entries=150, sequenceid=537, filesize=30.5 K 2024-11-12T07:50:42,618 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/B/5fe4e16a9ef944d5825ac5d29ee6ec30 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/5fe4e16a9ef944d5825ac5d29ee6ec30 2024-11-12T07:50:42,621 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/5fe4e16a9ef944d5825ac5d29ee6ec30, entries=150, sequenceid=537, filesize=12.0 K 2024-11-12T07:50:42,621 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/.tmp/C/3958c8ea8ccf49fea243d396808aa152 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3958c8ea8ccf49fea243d396808aa152 2024-11-12T07:50:42,624 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3958c8ea8ccf49fea243d396808aa152, entries=150, sequenceid=537, filesize=12.0 K 2024-11-12T07:50:42,625 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1c3bb6ad593aa45887e06136ab72eae4 in 1683ms, sequenceid=537, compaction requested=true 2024-11-12T07:50:42,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/a6f64658afa846e38defa57b9b9eb033, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/20dfe610905746adbab93c73cdcc4206, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/d0e98e92abcd43589ef7e4aa6f4e1421, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/1cf4448acbb4468f94b34b5b18309656, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/30761099303148679a494af9dd5b5f44, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4c6f68dc31f2484bb64dbe84f761164f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e49fe669105c42fea7c0322900a6c362, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4644933fd1ad40f98247a01c8253c365, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6604837e6f2e49dbbc87044e3ace4c76, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8857f8410b7545e88a53195721320966, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/df8df564f21c4ea090c82546d785d521, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6b224bbe27d34652b04637a07c6e67a5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e79b29f0774547b1b6a0b8f7812110e0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/51c27c1356d7434fa73eba8902e07725, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/dcf8c24c13be4fa9ba976102d08c99e1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/deb10096d5a4456eac46619ea68de819, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8ca87ebe98f74ae7beb2098cb86fe049, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/db01f16f8d5d4b7580647149c22cd301, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/97365be1ccfe4b28ad3ec1061e96a13a, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/eed7ca7aa04f4836b9ff43be35f5204d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/548f8d0e44d64bf0b4e2d60293e9b645, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e0998ff8c1944828b81645ef0b8d7656, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/c5acb9002575413eb2ff1319b7ebbc44] to archive 2024-11-12T07:50:42,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:50:42,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/a6f64658afa846e38defa57b9b9eb033 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/a6f64658afa846e38defa57b9b9eb033 2024-11-12T07:50:42,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/20dfe610905746adbab93c73cdcc4206 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/20dfe610905746adbab93c73cdcc4206 2024-11-12T07:50:42,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/d0e98e92abcd43589ef7e4aa6f4e1421 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/d0e98e92abcd43589ef7e4aa6f4e1421 2024-11-12T07:50:42,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/1cf4448acbb4468f94b34b5b18309656 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/1cf4448acbb4468f94b34b5b18309656 2024-11-12T07:50:42,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/30761099303148679a494af9dd5b5f44 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/30761099303148679a494af9dd5b5f44 2024-11-12T07:50:42,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4c6f68dc31f2484bb64dbe84f761164f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4c6f68dc31f2484bb64dbe84f761164f 2024-11-12T07:50:42,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e49fe669105c42fea7c0322900a6c362 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e49fe669105c42fea7c0322900a6c362 2024-11-12T07:50:42,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4644933fd1ad40f98247a01c8253c365 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/4644933fd1ad40f98247a01c8253c365 2024-11-12T07:50:42,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6604837e6f2e49dbbc87044e3ace4c76 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6604837e6f2e49dbbc87044e3ace4c76 2024-11-12T07:50:42,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8857f8410b7545e88a53195721320966 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8857f8410b7545e88a53195721320966 2024-11-12T07:50:42,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/df8df564f21c4ea090c82546d785d521 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/df8df564f21c4ea090c82546d785d521 2024-11-12T07:50:42,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6b224bbe27d34652b04637a07c6e67a5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/6b224bbe27d34652b04637a07c6e67a5 2024-11-12T07:50:42,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e79b29f0774547b1b6a0b8f7812110e0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e79b29f0774547b1b6a0b8f7812110e0 2024-11-12T07:50:42,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/51c27c1356d7434fa73eba8902e07725 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/51c27c1356d7434fa73eba8902e07725 2024-11-12T07:50:42,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/dcf8c24c13be4fa9ba976102d08c99e1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/dcf8c24c13be4fa9ba976102d08c99e1 2024-11-12T07:50:42,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/deb10096d5a4456eac46619ea68de819 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/deb10096d5a4456eac46619ea68de819 2024-11-12T07:50:42,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8ca87ebe98f74ae7beb2098cb86fe049 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/8ca87ebe98f74ae7beb2098cb86fe049 2024-11-12T07:50:42,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/db01f16f8d5d4b7580647149c22cd301 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/db01f16f8d5d4b7580647149c22cd301 2024-11-12T07:50:42,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/97365be1ccfe4b28ad3ec1061e96a13a to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/97365be1ccfe4b28ad3ec1061e96a13a 2024-11-12T07:50:42,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/eed7ca7aa04f4836b9ff43be35f5204d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/eed7ca7aa04f4836b9ff43be35f5204d 2024-11-12T07:50:42,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/548f8d0e44d64bf0b4e2d60293e9b645 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/548f8d0e44d64bf0b4e2d60293e9b645 2024-11-12T07:50:42,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e0998ff8c1944828b81645ef0b8d7656 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/e0998ff8c1944828b81645ef0b8d7656 2024-11-12T07:50:42,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/c5acb9002575413eb2ff1319b7ebbc44 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/c5acb9002575413eb2ff1319b7ebbc44 2024-11-12T07:50:42,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/846de35f5c9e4a398fe2053bc8bc8c1d, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d7540d6884b4f6e88a6a4a96c20989b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/16f6b61562404c7fbc1a5057b8905775, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e47509678e194824bacceb7fc72bdfd6, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/cdf18663793c4cd3a3897db9def49cd0, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/7a4f71d4bc0b4265a3d63e47b0a3c7c8, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d5e8be2d8294fddb63c3d65891c8f18, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e760044bb0c94c7aaa1d91ead9a87c23, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/d170d352c3924632987aa439e7c3c3b4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/4f247939d3324299a0bc381dd9c25e91, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/01afd0be28e84653bdf3c2b9640b77f2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/b9ddeabb429a421eb9758ce726a58a74, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/ee40c761eea74ca0ab2993dea54eb905, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/bceb50daf4bc49e0b2252c5d41013896, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f27eef36cba1403887aa7f41d60ff9cd, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/86b2e45ef5ab40daacdda7f6aba2c986, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/97e7ff3ce0484b0780a6d41ffa6ef159, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/201c1a1f4ef140d7a989e17f3c0d8536, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/8943f496ecc44e279c5fdd51f1dc4d8e, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/048ade6a02684896892a8380f41a9063, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e42a3bebc05a43ecaa10ad9b2ca03d6b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f002ab61eb48482cb9aeacf02f289013, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/116c8e5a3d254ff18f52e8fbc705f7ed] to archive 2024-11-12T07:50:42,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:50:42,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/846de35f5c9e4a398fe2053bc8bc8c1d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/846de35f5c9e4a398fe2053bc8bc8c1d 2024-11-12T07:50:42,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d7540d6884b4f6e88a6a4a96c20989b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d7540d6884b4f6e88a6a4a96c20989b 2024-11-12T07:50:42,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/16f6b61562404c7fbc1a5057b8905775 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/16f6b61562404c7fbc1a5057b8905775 2024-11-12T07:50:42,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e47509678e194824bacceb7fc72bdfd6 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e47509678e194824bacceb7fc72bdfd6 2024-11-12T07:50:42,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/cdf18663793c4cd3a3897db9def49cd0 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/cdf18663793c4cd3a3897db9def49cd0 2024-11-12T07:50:42,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/7a4f71d4bc0b4265a3d63e47b0a3c7c8 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/7a4f71d4bc0b4265a3d63e47b0a3c7c8 2024-11-12T07:50:42,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d5e8be2d8294fddb63c3d65891c8f18 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/9d5e8be2d8294fddb63c3d65891c8f18 2024-11-12T07:50:42,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e760044bb0c94c7aaa1d91ead9a87c23 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e760044bb0c94c7aaa1d91ead9a87c23 2024-11-12T07:50:42,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/d170d352c3924632987aa439e7c3c3b4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/d170d352c3924632987aa439e7c3c3b4 2024-11-12T07:50:42,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/4f247939d3324299a0bc381dd9c25e91 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/4f247939d3324299a0bc381dd9c25e91 2024-11-12T07:50:42,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/01afd0be28e84653bdf3c2b9640b77f2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/01afd0be28e84653bdf3c2b9640b77f2 2024-11-12T07:50:42,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/b9ddeabb429a421eb9758ce726a58a74 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/b9ddeabb429a421eb9758ce726a58a74 2024-11-12T07:50:42,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/ee40c761eea74ca0ab2993dea54eb905 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/ee40c761eea74ca0ab2993dea54eb905 2024-11-12T07:50:42,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/bceb50daf4bc49e0b2252c5d41013896 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/bceb50daf4bc49e0b2252c5d41013896 2024-11-12T07:50:42,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f27eef36cba1403887aa7f41d60ff9cd to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f27eef36cba1403887aa7f41d60ff9cd 2024-11-12T07:50:42,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/86b2e45ef5ab40daacdda7f6aba2c986 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/86b2e45ef5ab40daacdda7f6aba2c986 2024-11-12T07:50:42,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/97e7ff3ce0484b0780a6d41ffa6ef159 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/97e7ff3ce0484b0780a6d41ffa6ef159 2024-11-12T07:50:42,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/201c1a1f4ef140d7a989e17f3c0d8536 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/201c1a1f4ef140d7a989e17f3c0d8536 2024-11-12T07:50:42,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/8943f496ecc44e279c5fdd51f1dc4d8e to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/8943f496ecc44e279c5fdd51f1dc4d8e 2024-11-12T07:50:42,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/048ade6a02684896892a8380f41a9063 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/048ade6a02684896892a8380f41a9063 2024-11-12T07:50:42,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e42a3bebc05a43ecaa10ad9b2ca03d6b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/e42a3bebc05a43ecaa10ad9b2ca03d6b 2024-11-12T07:50:42,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f002ab61eb48482cb9aeacf02f289013 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/f002ab61eb48482cb9aeacf02f289013 2024-11-12T07:50:42,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/116c8e5a3d254ff18f52e8fbc705f7ed to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/116c8e5a3d254ff18f52e8fbc705f7ed 2024-11-12T07:50:42,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ae447f8cd972402996186d904298daa2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/088c290d1a51409a97e8be48270d21b3, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3502587c35ac47e1b1d1a5cd718ba3b9, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/77115b55f66a414b8c0ec1560075f395, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/f28dcfb7eaf84ddabe64d511da9c0716, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/489f2083d899437f9aa83ddcf7446645, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9cdc3e9a00af46d1a539535ec1558486, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/8278f07b7d474954a43ba5553aa5ff41, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b40a4ce688b74f0faae04af50f7345f5, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/328c2013acbb4bdb9dd04bd4d227d97f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/522fc602c7d44fe18352a329bce532b1, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/13b1b17560844e3e8a8468864f005a48, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a62d602b1e0b4ba78efdcfaaf238324c, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b274cae46d544b24a87ad0c4de0f3721, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/10ed856d12d34356b071af36a67a1061, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/e6db49fa3196429681195b82b1c6f31f, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a05ce3bb6b0b49b9912aaeb1c8762533, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ba503536890e497aa1f02a4446bcbbd4, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/95ac32ef83ad4b64a5a27a50ec8c12b2, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9197ce453e344264b3ebf37713138e9b, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/6c7d9fb428ed4ca5a549a20c09f92f82, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/5e607849e9174612a1556dca88ad2fe7] to archive 2024-11-12T07:50:42,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T07:50:42,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ae447f8cd972402996186d904298daa2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ae447f8cd972402996186d904298daa2 2024-11-12T07:50:42,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/088c290d1a51409a97e8be48270d21b3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/088c290d1a51409a97e8be48270d21b3 2024-11-12T07:50:42,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3502587c35ac47e1b1d1a5cd718ba3b9 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3502587c35ac47e1b1d1a5cd718ba3b9 2024-11-12T07:50:42,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/77115b55f66a414b8c0ec1560075f395 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/77115b55f66a414b8c0ec1560075f395 2024-11-12T07:50:42,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/f28dcfb7eaf84ddabe64d511da9c0716 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/f28dcfb7eaf84ddabe64d511da9c0716 2024-11-12T07:50:42,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/489f2083d899437f9aa83ddcf7446645 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/489f2083d899437f9aa83ddcf7446645 2024-11-12T07:50:42,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9cdc3e9a00af46d1a539535ec1558486 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9cdc3e9a00af46d1a539535ec1558486 2024-11-12T07:50:42,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/7c0fdd7e648b4ab6b71ec6fc6baff4fe 2024-11-12T07:50:42,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/8278f07b7d474954a43ba5553aa5ff41 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/8278f07b7d474954a43ba5553aa5ff41 2024-11-12T07:50:42,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b40a4ce688b74f0faae04af50f7345f5 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b40a4ce688b74f0faae04af50f7345f5 2024-11-12T07:50:42,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/328c2013acbb4bdb9dd04bd4d227d97f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/328c2013acbb4bdb9dd04bd4d227d97f 2024-11-12T07:50:42,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/522fc602c7d44fe18352a329bce532b1 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/522fc602c7d44fe18352a329bce532b1 2024-11-12T07:50:42,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/13b1b17560844e3e8a8468864f005a48 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/13b1b17560844e3e8a8468864f005a48 2024-11-12T07:50:42,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a62d602b1e0b4ba78efdcfaaf238324c to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a62d602b1e0b4ba78efdcfaaf238324c 2024-11-12T07:50:42,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b274cae46d544b24a87ad0c4de0f3721 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b274cae46d544b24a87ad0c4de0f3721 2024-11-12T07:50:42,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/10ed856d12d34356b071af36a67a1061 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/10ed856d12d34356b071af36a67a1061 2024-11-12T07:50:42,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/e6db49fa3196429681195b82b1c6f31f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/e6db49fa3196429681195b82b1c6f31f 2024-11-12T07:50:42,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a05ce3bb6b0b49b9912aaeb1c8762533 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/a05ce3bb6b0b49b9912aaeb1c8762533 2024-11-12T07:50:42,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ba503536890e497aa1f02a4446bcbbd4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/ba503536890e497aa1f02a4446bcbbd4 2024-11-12T07:50:42,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/95ac32ef83ad4b64a5a27a50ec8c12b2 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/95ac32ef83ad4b64a5a27a50ec8c12b2 2024-11-12T07:50:42,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9197ce453e344264b3ebf37713138e9b to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/9197ce453e344264b3ebf37713138e9b 2024-11-12T07:50:42,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/6c7d9fb428ed4ca5a549a20c09f92f82 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/6c7d9fb428ed4ca5a549a20c09f92f82 2024-11-12T07:50:42,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/5e607849e9174612a1556dca88ad2fe7 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/5e607849e9174612a1556dca88ad2fe7 2024-11-12T07:50:42,682 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/recovered.edits/540.seqid, newMaxSeqId=540, maxSeqId=4 2024-11-12T07:50:42,683 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. 2024-11-12T07:50:42,683 DEBUG [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 1c3bb6ad593aa45887e06136ab72eae4: Waiting for close lock at 1731397840941Running coprocessor pre-close hooks at 1731397840941Disabling compacts and flushes for region at 1731397840941Disabling writes for close at 1731397840941Obtaining lock to block concurrent updates at 1731397840941Preparing flush snapshotting stores in 1c3bb6ad593aa45887e06136ab72eae4 at 1731397840941Finished memstore snapshotting TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731397840942 (+1 ms)Flushing stores of TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4. at 1731397840942Flushing 1c3bb6ad593aa45887e06136ab72eae4/A: creating writer at 1731397840942Flushing mob file 1c3bb6ad593aa45887e06136ab72eae4/A: appending metadata at 1731397840945 (+3 ms)Flushing mob file 1c3bb6ad593aa45887e06136ab72eae4/A: closing flushed file at 1731397840945Flushing 1c3bb6ad593aa45887e06136ab72eae4/A: appending metadata at 1731397841353 (+408 ms)Flushing 1c3bb6ad593aa45887e06136ab72eae4/A: closing flushed file at 1731397841353Flushing 1c3bb6ad593aa45887e06136ab72eae4/B: creating writer at 1731397841796 (+443 ms)Flushing 1c3bb6ad593aa45887e06136ab72eae4/B: appending metadata at 1731397841799 (+3 ms)Flushing 1c3bb6ad593aa45887e06136ab72eae4/B: closing flushed file at 1731397841799Flushing 1c3bb6ad593aa45887e06136ab72eae4/C: creating writer at 1731397842206 (+407 ms)Flushing 1c3bb6ad593aa45887e06136ab72eae4/C: appending metadata at 1731397842207 (+1 ms)Flushing 1c3bb6ad593aa45887e06136ab72eae4/C: closing flushed file at 1731397842207Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43b16855: reopening flushed file at 1731397842614 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6549b: reopening flushed file at 1731397842618 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18ebb686: reopening flushed file at 1731397842621 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 1c3bb6ad593aa45887e06136ab72eae4 in 1683ms, sequenceid=537, compaction requested=true at 1731397842625 (+4 ms)Writing region close event to WAL at 1731397842680 (+55 ms)Running coprocessor post-close hooks at 1731397842683 (+3 ms)Closed at 1731397842683 2024-11-12T07:50:42,685 INFO [RS_CLOSE_REGION-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,685 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=1c3bb6ad593aa45887e06136ab72eae4, regionState=CLOSED 2024-11-12T07:50:42,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 because future has completed 2024-11-12T07:50:42,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-12T07:50:42,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c3bb6ad593aa45887e06136ab72eae4, server=f9fc96254ef8,44299,1731397650797 in 1.8980 sec 2024-11-12T07:50:42,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-12T07:50:42,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1c3bb6ad593aa45887e06136ab72eae4, UNASSIGN in 1.9010 sec 2024-11-12T07:50:42,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-12T07:50:42,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9040 sec 2024-11-12T07:50:42,692 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731397842691"}]},"ts":"1731397842691"} 2024-11-12T07:50:42,693 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T07:50:42,693 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T07:50:42,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9120 sec 2024-11-12T07:50:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T07:50:42,938 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:42,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-12T07:50:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:42,941 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T07:50:42,941 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:42,944 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,945 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C, FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/recovered.edits] 2024-11-12T07:50:42,947 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/05ca74c174cc4458b4e66ede929cd409 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/05ca74c174cc4458b4e66ede929cd409 2024-11-12T07:50:42,948 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/b059d470a5f44589b4fbd32a1ba82b16 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/b059d470a5f44589b4fbd32a1ba82b16 2024-11-12T07:50:42,949 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/ccb96426326c4aae9074a8fbe74bbf3f to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/A/ccb96426326c4aae9074a8fbe74bbf3f 2024-11-12T07:50:42,950 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/5fe4e16a9ef944d5825ac5d29ee6ec30 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/5fe4e16a9ef944d5825ac5d29ee6ec30 2024-11-12T07:50:42,951 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/885933439aaf40e7892fc72882ba15f3 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/885933439aaf40e7892fc72882ba15f3 2024-11-12T07:50:42,952 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/adf4342efb974ff4aa2761b82288d68d to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/B/adf4342efb974ff4aa2761b82288d68d 2024-11-12T07:50:42,953 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3958c8ea8ccf49fea243d396808aa152 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3958c8ea8ccf49fea243d396808aa152 2024-11-12T07:50:42,954 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3b323c1c738b411e983c692ed7574e44 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/3b323c1c738b411e983c692ed7574e44 2024-11-12T07:50:42,954 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b0392645cb2b451d98f1cad92f358148 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/C/b0392645cb2b451d98f1cad92f358148 2024-11-12T07:50:42,956 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/recovered.edits/540.seqid to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4/recovered.edits/540.seqid 2024-11-12T07:50:42,957 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/default/TestAcidGuarantees/1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,957 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T07:50:42,957 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T07:50:42,958 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T07:50:42,960 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112001220291e114620b1b2c977147b00f9_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112001220291e114620b1b2c977147b00f9_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,961 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120f9ff2d00d56447e888c4a4aa3c4d550_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120f9ff2d00d56447e888c4a4aa3c4d550_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,961 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121a2025eaa2d746fda578cae610148e82_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121a2025eaa2d746fda578cae610148e82_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,962 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c801c420ba647628eb264d78642736e_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122c801c420ba647628eb264d78642736e_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,963 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122dd53deaf1474b57942384f6fc7f7096_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122dd53deaf1474b57942384f6fc7f7096_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,964 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111241212d362d254c69b29993f4d8111dae_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111241212d362d254c69b29993f4d8111dae_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,965 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125e2c23688fed4e5c9216e271bf97788e_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125e2c23688fed4e5c9216e271bf97788e_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,966 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111270c50aeddb0f47c29814c43f87c8573c_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111270c50aeddb0f47c29814c43f87c8573c_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,967 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e86ebd6576e4b7d95ccf43a0eb3637f_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411127e86ebd6576e4b7d95ccf43a0eb3637f_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,967 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111287cbf844ea734e9d8f7f3a123e889370_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111287cbf844ea734e9d8f7f3a123e889370_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,968 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112988ba9e44c2e42ee97629ce6577ce3cb_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112988ba9e44c2e42ee97629ce6577ce3cb_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,969 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129cef468039e74424b6567c134a9c48bc_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129cef468039e74424b6567c134a9c48bc_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,970 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b59ead9915fd49139265ed8d3dfc1ef3_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b59ead9915fd49139265ed8d3dfc1ef3_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,970 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c667df0de447435d8149b665bf56d245_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c667df0de447435d8149b665bf56d245_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,971 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d45e24a9ce1f49dcb5d5d2dfe5ae4e8c_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d45e24a9ce1f49dcb5d5d2dfe5ae4e8c_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,972 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112daf0e33a1d364408bd3b8f024f63ae09_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112daf0e33a1d364408bd3b8f024f63ae09_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,972 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e295eaa2afd343b380399dd676ac35b3_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e295eaa2afd343b380399dd676ac35b3_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,973 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2d034bce3a848a4a7f0f15eef3e4c09_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2d034bce3a848a4a7f0f15eef3e4c09_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,974 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2fd7c6f3e334de5a5bea60b7686311f_1c3bb6ad593aa45887e06136ab72eae4 to hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f2fd7c6f3e334de5a5bea60b7686311f_1c3bb6ad593aa45887e06136ab72eae4 2024-11-12T07:50:42,974 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T07:50:42,976 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:42,978 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T07:50:42,980 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T07:50:42,981 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:42,981 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T07:50:42,981 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731397842981"}]},"ts":"9223372036854775807"} 2024-11-12T07:50:42,982 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T07:50:42,982 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1c3bb6ad593aa45887e06136ab72eae4, NAME => 'TestAcidGuarantees,,1731397809656.1c3bb6ad593aa45887e06136ab72eae4.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T07:50:42,983 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T07:50:42,983 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731397842983"}]},"ts":"9223372036854775807"} 2024-11-12T07:50:42,984 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T07:50:42,985 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T07:50:42,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-12T07:50:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T07:50:43,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T07:50:43,051 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T07:50:43,060 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 289) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6261 (was 5784) - AvailableMemoryMB LEAK? - 2024-11-12T07:50:43,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T07:50:43,060 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:50:43,060 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:43,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:43,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:43,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T07:50:43,061 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T07:50:43,061 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2038551008, stopped=false 2024-11-12T07:50:43,061 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f9fc96254ef8,43319,1731397650023 2024-11-12T07:50:43,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:43,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:43,168 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T07:50:43,169 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T07:50:43,169 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:43,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:43,169 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f9fc96254ef8,44299,1731397650797' ***** 2024-11-12T07:50:43,169 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T07:50:43,170 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T07:50:43,170 INFO [RS:0;f9fc96254ef8:44299 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T07:50:43,170 INFO [RS:0;f9fc96254ef8:44299 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T07:50:43,170 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(959): stopping server f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:43,170 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:50:43,170 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T07:50:43,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T07:50:43,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:43,171 INFO [RS:0;f9fc96254ef8:44299 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f9fc96254ef8:44299. 2024-11-12T07:50:43,171 DEBUG [RS:0;f9fc96254ef8:44299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T07:50:43,171 DEBUG [RS:0;f9fc96254ef8:44299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:43,171 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T07:50:43,171 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T07:50:43,171 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T07:50:43,171 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T07:50:43,172 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:43,172 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T07:50:43,172 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T07:50:43,172 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T07:50:43,172 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T07:50:43,172 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T07:50:43,172 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T07:50:43,173 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T07:50:43,173 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T07:50:43,173 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T07:50:43,173 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-12T07:50:43,197 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/info/5ee893e3068b4280af8e0402efae04b3 is 95, key is TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b./info:/1731397683067/DeleteFamily/seqid=0 2024-11-12T07:50:43,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742418_1594 (size=6514) 2024-11-12T07:50:43,222 INFO [regionserver/f9fc96254ef8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:50:43,373 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T07:50:43,573 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T07:50:43,608 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/info/5ee893e3068b4280af8e0402efae04b3 2024-11-12T07:50:43,626 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/ns/134ec12cf56f450791f19bc2185b96f9 is 93, key is TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b./ns:/1731397683027/DeleteFamily/seqid=0 2024-11-12T07:50:43,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742419_1595 (size=5906) 2024-11-12T07:50:43,773 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T07:50:43,845 INFO [regionserver/f9fc96254ef8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T07:50:43,845 INFO [regionserver/f9fc96254ef8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T07:50:43,973 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T07:50:44,029 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/ns/134ec12cf56f450791f19bc2185b96f9 2024-11-12T07:50:44,045 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/rep_barrier/75c2892756e349768ec8bce9d7f320c6 is 102, key is TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b./rep_barrier:/1731397683027/DeleteFamily/seqid=0 2024-11-12T07:50:44,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742420_1596 (size=6025) 2024-11-12T07:50:44,173 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T07:50:44,374 DEBUG [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T07:50:44,457 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/rep_barrier/75c2892756e349768ec8bce9d7f320c6 2024-11-12T07:50:44,480 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/table/d7624e35aa3a41dda84e790df887fe3a is 96, key is TestAcidGuarantees,,1731397655411.609f0702d0c4b8c7ee5a4c9652b6127b./table:/1731397683027/DeleteFamily/seqid=0 2024-11-12T07:50:44,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742421_1597 (size=5936) 2024-11-12T07:50:44,488 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/table/d7624e35aa3a41dda84e790df887fe3a 2024-11-12T07:50:44,493 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/info/5ee893e3068b4280af8e0402efae04b3 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/info/5ee893e3068b4280af8e0402efae04b3 2024-11-12T07:50:44,497 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/info/5ee893e3068b4280af8e0402efae04b3, entries=12, sequenceid=90, filesize=6.4 K 2024-11-12T07:50:44,498 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/ns/134ec12cf56f450791f19bc2185b96f9 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/ns/134ec12cf56f450791f19bc2185b96f9 2024-11-12T07:50:44,501 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/ns/134ec12cf56f450791f19bc2185b96f9, entries=8, sequenceid=90, filesize=5.8 K 2024-11-12T07:50:44,502 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/rep_barrier/75c2892756e349768ec8bce9d7f320c6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/rep_barrier/75c2892756e349768ec8bce9d7f320c6 2024-11-12T07:50:44,506 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/rep_barrier/75c2892756e349768ec8bce9d7f320c6, entries=6, sequenceid=90, filesize=5.9 K 2024-11-12T07:50:44,507 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/.tmp/table/d7624e35aa3a41dda84e790df887fe3a as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/table/d7624e35aa3a41dda84e790df887fe3a 2024-11-12T07:50:44,510 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/table/d7624e35aa3a41dda84e790df887fe3a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-12T07:50:44,511 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1338ms, sequenceid=90, compaction requested=false 2024-11-12T07:50:44,523 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-12T07:50:44,523 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T07:50:44,523 INFO [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T07:50:44,524 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731397843172Running coprocessor pre-close hooks at 1731397843172Disabling compacts and flushes for region at 1731397843172Disabling writes for close at 1731397843173 (+1 ms)Obtaining lock to block concurrent updates at 1731397843173Preparing flush snapshotting stores in 1588230740 at 1731397843173Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731397843173Flushing stores of hbase:meta,,1.1588230740 at 1731397843174 (+1 ms)Flushing 1588230740/info: creating writer at 1731397843174Flushing 1588230740/info: appending metadata at 1731397843197 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731397843197Flushing 1588230740/ns: creating writer at 1731397843612 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731397843625 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731397843625Flushing 1588230740/rep_barrier: creating writer at 1731397844032 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731397844045 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731397844045Flushing 1588230740/table: creating writer at 1731397844460 (+415 ms)Flushing 1588230740/table: appending metadata at 1731397844480 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731397844480Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e07be94: reopening flushed file at 1731397844492 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11f2195c: reopening flushed file at 1731397844497 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1788194b: reopening flushed file at 1731397844502 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a2fc562: reopening flushed file at 1731397844506 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1338ms, sequenceid=90, compaction requested=false at 1731397844511 (+5 ms)Writing region close event to WAL at 1731397844518 (+7 ms)Running coprocessor post-close hooks at 1731397844523 (+5 ms)Closed at 1731397844523 2024-11-12T07:50:44,524 DEBUG [RS_CLOSE_META-regionserver/f9fc96254ef8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T07:50:44,574 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(976): stopping server f9fc96254ef8,44299,1731397650797; all regions closed. 2024-11-12T07:50:44,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741834_1010 (size=25344) 2024-11-12T07:50:44,580 DEBUG [RS:0;f9fc96254ef8:44299 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/oldWALs 2024-11-12T07:50:44,580 INFO [RS:0;f9fc96254ef8:44299 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C44299%2C1731397650797.meta:.meta(num 1731397655015) 2024-11-12T07:50:44,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741832_1008 (size=16702091) 2024-11-12T07:50:44,588 DEBUG [RS:0;f9fc96254ef8:44299 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/oldWALs 2024-11-12T07:50:44,588 INFO [RS:0;f9fc96254ef8:44299 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f9fc96254ef8%2C44299%2C1731397650797:(num 1731397653971) 2024-11-12T07:50:44,588 DEBUG [RS:0;f9fc96254ef8:44299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T07:50:44,588 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T07:50:44,588 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:50:44,588 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.ChoreService(370): Chore service for: regionserver/f9fc96254ef8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T07:50:44,589 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:50:44,589 INFO [regionserver/f9fc96254ef8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:50:44,589 INFO [RS:0;f9fc96254ef8:44299 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44299 2024-11-12T07:50:44,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f9fc96254ef8,44299,1731397650797 2024-11-12T07:50:44,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T07:50:44,631 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:50:44,632 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f9fc96254ef8,44299,1731397650797] 2024-11-12T07:50:44,647 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f9fc96254ef8,44299,1731397650797 already deleted, retry=false 2024-11-12T07:50:44,648 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f9fc96254ef8,44299,1731397650797 expired; onlineServers=0 2024-11-12T07:50:44,648 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f9fc96254ef8,43319,1731397650023' ***** 2024-11-12T07:50:44,648 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T07:50:44,648 INFO [M:0;f9fc96254ef8:43319 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T07:50:44,648 INFO [M:0;f9fc96254ef8:43319 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T07:50:44,648 DEBUG [M:0;f9fc96254ef8:43319 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T07:50:44,648 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T07:50:44,648 DEBUG [M:0;f9fc96254ef8:43319 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T07:50:44,648 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397653676 {}] cleaner.HFileCleaner(306): Exit Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.small.0-1731397653676,5,FailOnTimeoutGroup] 2024-11-12T07:50:44,648 DEBUG [master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397653673 {}] cleaner.HFileCleaner(306): Exit Thread[master/f9fc96254ef8:0:becomeActiveMaster-HFileCleaner.large.0-1731397653673,5,FailOnTimeoutGroup] 2024-11-12T07:50:44,648 INFO [M:0;f9fc96254ef8:43319 {}] hbase.ChoreService(370): Chore service for: master/f9fc96254ef8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T07:50:44,648 INFO [M:0;f9fc96254ef8:43319 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T07:50:44,649 DEBUG [M:0;f9fc96254ef8:43319 {}] master.HMaster(1795): Stopping service threads 2024-11-12T07:50:44,649 INFO [M:0;f9fc96254ef8:43319 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T07:50:44,649 INFO [M:0;f9fc96254ef8:43319 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T07:50:44,649 ERROR [M:0;f9fc96254ef8:43319 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:34321 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:34321,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-12T07:50:44,649 INFO [M:0;f9fc96254ef8:43319 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T07:50:44,650 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T07:50:44,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T07:50:44,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T07:50:44,664 DEBUG [M:0;f9fc96254ef8:43319 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-12T07:50:44,664 DEBUG [M:0;f9fc96254ef8:43319 {}] master.ActiveMasterManager(353): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-12T07:50:44,665 INFO [M:0;f9fc96254ef8:43319 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/.lastflushedseqids 2024-11-12T07:50:44,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742422_1598 (size=72) 2024-11-12T07:50:44,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:44,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44299-0x1012f4d15df0001, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:44,740 INFO [RS:0;f9fc96254ef8:44299 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:50:44,740 INFO [RS:0;f9fc96254ef8:44299 {}] regionserver.HRegionServer(1031): Exiting; stopping=f9fc96254ef8,44299,1731397650797; zookeeper connection closed. 2024-11-12T07:50:44,741 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1b709fe {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1b709fe 2024-11-12T07:50:44,741 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T07:50:45,074 INFO [M:0;f9fc96254ef8:43319 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T07:50:45,074 INFO [M:0;f9fc96254ef8:43319 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T07:50:45,074 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T07:50:45,074 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:45,074 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:45,074 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T07:50:45,074 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:45,074 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=749.42 KB heapSize=916.13 KB 2024-11-12T07:50:45,091 DEBUG [M:0;f9fc96254ef8:43319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/172414b0daaa433f98f534bff9549c19 is 82, key is hbase:meta,,1/info:regioninfo/1731397655092/Put/seqid=0 2024-11-12T07:50:45,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742423_1599 (size=5672) 2024-11-12T07:50:45,099 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2057 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/172414b0daaa433f98f534bff9549c19 2024-11-12T07:50:45,143 DEBUG [M:0;f9fc96254ef8:43319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b04fbf16ee2f410eb77b266cf08df7f6 is 2480, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1731397686264/Put/seqid=0 2024-11-12T07:50:45,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742424_1600 (size=44022) 2024-11-12T07:50:45,167 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=748.87 KB at sequenceid=2057 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b04fbf16ee2f410eb77b266cf08df7f6 2024-11-12T07:50:45,174 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b04fbf16ee2f410eb77b266cf08df7f6 2024-11-12T07:50:45,198 DEBUG [M:0;f9fc96254ef8:43319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb53387aef14495fb60ee1db403d5c0d is 69, key is f9fc96254ef8,44299,1731397650797/rs:state/1731397653716/Put/seqid=0 2024-11-12T07:50:45,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073742425_1601 (size=5156) 2024-11-12T07:50:45,234 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2057 (bloomFilter=true), to=hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb53387aef14495fb60ee1db403d5c0d 2024-11-12T07:50:45,239 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/172414b0daaa433f98f534bff9549c19 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/172414b0daaa433f98f534bff9549c19 2024-11-12T07:50:45,243 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/172414b0daaa433f98f534bff9549c19, entries=8, sequenceid=2057, filesize=5.5 K 2024-11-12T07:50:45,244 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b04fbf16ee2f410eb77b266cf08df7f6 as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b04fbf16ee2f410eb77b266cf08df7f6 2024-11-12T07:50:45,248 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b04fbf16ee2f410eb77b266cf08df7f6 2024-11-12T07:50:45,248 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b04fbf16ee2f410eb77b266cf08df7f6, entries=162, sequenceid=2057, filesize=43.0 K 2024-11-12T07:50:45,249 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb53387aef14495fb60ee1db403d5c0d as hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fb53387aef14495fb60ee1db403d5c0d 2024-11-12T07:50:45,267 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34321/user/jenkins/test-data/3ebb48f0-9b85-d22d-b6f8-27294f01c67f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fb53387aef14495fb60ee1db403d5c0d, entries=1, sequenceid=2057, filesize=5.0 K 2024-11-12T07:50:45,268 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(3140): Finished flush of dataSize ~749.42 KB/767411, heapSize ~915.84 KB/937816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 194ms, sequenceid=2057, compaction requested=false 2024-11-12T07:50:45,299 INFO [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T07:50:45,299 DEBUG [M:0;f9fc96254ef8:43319 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731397845074Disabling compacts and flushes for region at 1731397845074Disabling writes for close at 1731397845074Obtaining lock to block concurrent updates at 1731397845074Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731397845074Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=767411, getHeapSize=938056, getOffHeapSize=0, getCellsCount=2244 at 1731397845075 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731397845075Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731397845075Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731397845090 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731397845090Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731397845105 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731397845142 (+37 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731397845142Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731397845174 (+32 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731397845198 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731397845198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2131187: reopening flushed file at 1731397845238 (+40 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ded98c2: reopening flushed file at 1731397845243 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48288336: reopening flushed file at 1731397845248 (+5 ms)Finished flush of dataSize ~749.42 KB/767411, heapSize ~915.84 KB/937816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 194ms, sequenceid=2057, compaction requested=false at 1731397845268 (+20 ms)Writing region close event to WAL at 1731397845299 (+31 ms)Closed at 1731397845299 2024-11-12T07:50:45,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40253 is added to blk_1073741830_1006 (size=902283) 2024-11-12T07:50:45,315 INFO [M:0;f9fc96254ef8:43319 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T07:50:45,315 INFO [M:0;f9fc96254ef8:43319 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43319 2024-11-12T07:50:45,315 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T07:50:45,315 INFO [M:0;f9fc96254ef8:43319 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T07:50:45,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:45,481 INFO [M:0;f9fc96254ef8:43319 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T07:50:45,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43319-0x1012f4d15df0000, quorum=127.0.0.1:63742, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T07:50:45,485 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T07:50:45,487 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T07:50:45,487 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T07:50:45,487 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T07:50:45,487 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/hadoop.log.dir/,STOPPED} 2024-11-12T07:50:45,490 WARN [BP-611431002-172.17.0.2-1731397646649 heartbeating to localhost/127.0.0.1:34321 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T07:50:45,490 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T07:50:45,490 WARN [BP-611431002-172.17.0.2-1731397646649 heartbeating to localhost/127.0.0.1:34321 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-611431002-172.17.0.2-1731397646649 (Datanode Uuid e2a7e571-c659-4a13-8f26-ad75541ae184) service to localhost/127.0.0.1:34321 2024-11-12T07:50:45,490 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T07:50:45,492 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/cluster_8b863ce8-e81b-609c-c8dc-bd60e08d8037/data/data1/current/BP-611431002-172.17.0.2-1731397646649 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T07:50:45,492 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/cluster_8b863ce8-e81b-609c-c8dc-bd60e08d8037/data/data2/current/BP-611431002-172.17.0.2-1731397646649 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T07:50:45,493 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T07:50:45,499 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T07:50:45,500 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T07:50:45,500 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T07:50:45,500 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T07:50:45,501 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6edea3b1-f771-f06c-5a86-0ca50f1cf4f4/hadoop.log.dir/,STOPPED} 2024-11-12T07:50:45,520 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T07:50:45,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down